{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "2a8e89d4",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/gpfs/radev/home/tl688/.conda/envs/evoagentx/lib/python3.11/site-packages/PyPDF2/__init__.py:21: DeprecationWarning: PyPDF2 is deprecated. Please move to the pypdf library instead.\n",
      "  warnings.warn(\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "\n",
    "from dotenv import load_dotenv\n",
    "\n",
    "from evoagentx.agents.agent_manager import AgentManager\n",
    "from evoagentx.benchmark import HotPotQA\n",
    "from evoagentx.core.callbacks import suppress_logger_info\n",
    "from evoagentx.core.logging import logger\n",
    "from evoagentx.evaluators import Evaluator\n",
    "from evoagentx.models import OpenAILLM, OpenAILLMConfig\n",
    "from evoagentx.optimizers import TextGradOptimizer\n",
    "from evoagentx.prompts import StringTemplate\n",
    "from evoagentx.workflow import SequentialWorkFlowGraph\n",
    "from dotenv import load_dotenv\n",
    "\n",
    "from evoagentx.agents.agent_manager import AgentManager\n",
    "from evoagentx.benchmark import MBPP\n",
    "from evoagentx.core.callbacks import suppress_logger_info\n",
    "from evoagentx.core.logging import logger\n",
    "from evoagentx.evaluators import Evaluator\n",
    "from evoagentx.models import OpenAILLM, OpenAILLMConfig\n",
    "from evoagentx.optimizers import TextGradOptimizer\n",
    "from evoagentx.prompts import StringTemplate\n",
    "from evoagentx.workflow import SequentialWorkFlowGraph\n",
    "\n",
    "from evoagentx.models import OpenAILLMConfig, OpenAILLM\n",
    "from evoagentx.workflow import SEWWorkFlowGraph, STRUCTUREWorkFlowGraph\n",
    "from evoagentx.agents import AgentManager\n",
    "from evoagentx.benchmark import HumanEval,AFlowMBPP\n",
    "from evoagentx.evaluators import Evaluator \n",
    "from evoagentx.optimizers import SEWOptimizer, STRUCTUREOptimizer\n",
    "from evoagentx.optimizers.structure_optimizer import STRUCTUREWorkFlowScheme\n",
    "from evoagentx.core.callbacks import suppress_logger_info\n",
    "\n",
    "from evoagentx.models import OpenAILLMConfig, OpenAILLM,AzureOpenAIConfig,LiteLLMConfig,LiteLLM\n",
    "from evoagentx.workflow import SEWWorkFlowGraph \n",
    "from evoagentx.agents import AgentManager\n",
    "from evoagentx.benchmark import MBPPPLUS, AFlowMBPPPLUS\n",
    "from evoagentx.evaluators import Evaluator \n",
    "from evoagentx.optimizers import SEWOptimizer \n",
    "from evoagentx.core.callbacks import suppress_logger_info\n",
    "from evoagentx.benchmark import HumanEvalPLUS\n",
    "from evoagentx.benchmark import SciCode\n",
    "from copy import deepcopy\n",
    "\n",
    "import nest_asyncio\n",
    "nest_asyncio.apply()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "54f40417",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "\n",
    "from dotenv import load_dotenv\n",
    "\n",
    "from evoagentx.agents.agent_manager import AgentManager\n",
    "from evoagentx.benchmark import HotPotQA\n",
    "from evoagentx.core.callbacks import suppress_logger_info\n",
    "from evoagentx.core.logging import logger\n",
    "from evoagentx.evaluators import Evaluator\n",
    "from evoagentx.models import OpenAILLM, OpenAILLMConfig\n",
    "from evoagentx.optimizers import TextGradOptimizer\n",
    "from evoagentx.prompts import StringTemplate\n",
    "from evoagentx.workflow import SequentialWorkFlowGraph\n",
    "from dotenv import load_dotenv\n",
    "\n",
    "from evoagentx.agents.agent_manager import AgentManager\n",
    "from evoagentx.benchmark import MBPP\n",
    "from evoagentx.core.callbacks import suppress_logger_info\n",
    "from evoagentx.core.logging import logger\n",
    "from evoagentx.evaluators import Evaluator\n",
    "from evoagentx.models import OpenAILLM, OpenAILLMConfig\n",
    "from evoagentx.optimizers import TextGradOptimizer\n",
    "from evoagentx.prompts import StringTemplate\n",
    "from evoagentx.workflow import SequentialWorkFlowGraph\n",
    "\n",
    "from evoagentx.models import OpenAILLMConfig, OpenAILLM\n",
    "from evoagentx.workflow import SEWWorkFlowGraph, STRUCTUREWorkFlowGraph\n",
    "from evoagentx.agents import AgentManager\n",
    "from evoagentx.benchmark import HumanEval,AFlowMBPP\n",
    "from evoagentx.evaluators import Evaluator \n",
    "from evoagentx.optimizers import SEWOptimizer, STRUCTUREOptimizer\n",
    "from evoagentx.optimizers.structure_optimizer import STRUCTUREWorkFlowScheme\n",
    "from evoagentx.core.callbacks import suppress_logger_info\n",
    "\n",
    "from evoagentx.models import OpenAILLMConfig, OpenAILLM,AzureOpenAIConfig,LiteLLMConfig,LiteLLM\n",
    "from evoagentx.workflow import SEWWorkFlowGraph \n",
    "from evoagentx.agents import AgentManager\n",
    "from evoagentx.benchmark import MBPPPLUS, AFlowMBPPPLUS\n",
    "from evoagentx.evaluators import Evaluator \n",
    "from evoagentx.optimizers import SEWOptimizer \n",
    "from evoagentx.core.callbacks import suppress_logger_info\n",
    "from evoagentx.benchmark import HumanEvalPLUS\n",
    "from evoagentx.benchmark import SciCode\n",
    "from evoagentx.benchmark import PertQA\n",
    "from copy import deepcopy\n",
    "\n",
    "import nest_asyncio\n",
    "nest_asyncio.apply()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "54fa1aa5",
   "metadata": {},
   "outputs": [],
   "source": [
    "from evoagentx.benchmark import PertQA\n",
    "from copy import deepcopy\n",
    "\n",
    "import nest_asyncio\n",
    "nest_asyncio.apply()\n",
    "\n",
    "def collate_func(example: dict) -> dict:\n",
    "    problem = \"Question: {}\\n\\nAnswer:\".format(example[\"question\"])\n",
    "    return {\"problem\": problem}\n",
    "\n",
    "\n",
    "hotpotqa_graph_data = {\n",
    "    \"goal\": \"Answer the question based on the context. The answer should be a direct response to the question, without including explanations or reasoning.\",\n",
    "    \"tasks\": [\n",
    "        {\n",
    "            \"name\": \"answer_generate\",\n",
    "            \"description\": \"Answer the question based on the context.\",\n",
    "            \"inputs\": [\n",
    "                {\"name\": \"problem\", \"type\": \"str\", \"required\": True, \"description\": \"The problem to solve.\"}\n",
    "            ],\n",
    "            \"outputs\": [\n",
    "                {\"name\": \"answer\", \"type\": \"str\", \"required\": True, \"description\": \"The answer to the problem.\"}\n",
    "            ],\n",
    "            \"prompt_template\": StringTemplate(instruction=\"Think step by step to answer the question. You should explain your thinking process in the 'thought' field, and provide the final answer in the 'answer' field.\\nFormat your output in xml format, such as <thought>xxx</thought> and <answer>xxx</answer>.\"),\n",
    "            \"parse_mode\": \"xml\"\n",
    "        }\n",
    "    ] \n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "1ebace55",
   "metadata": {},
   "outputs": [],
   "source": [
    "api_key = \"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\"\n",
    "# Function to encode the image\n",
    "import os\n",
    "os.environ[\"OPENAI_API_KEY\"] = api_key\n",
    "OPENAI_API_KEY = os.getenv(\"OPENAI_API_KEY\")\n",
    "\n",
    "\n",
    "llm_config = OpenAILLMConfig(model=\"gpt-4o-mini-2024-07-18\", openai_key=OPENAI_API_KEY, top_p=0.85, temperature=0.2, frequency_penalty=0.0, presence_penalty=0.0)\n",
    "llm = OpenAILLM(config=llm_config)\n",
    "executor_llm = OpenAILLM(config=llm_config)\n",
    "optimizer_llm = OpenAILLM(config=llm_config)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "20e078fa",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[32m2026-01-04 09:01:26.892\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.benchmark.pertqa\u001b[0m:\u001b[36m_load_data_from_file\u001b[0m:\u001b[36m52\u001b[0m - \u001b[1mloading HotPotQA data from /home/tl688/pitl688/selfevolve/EvoAgentX/examples/pertqa/reploge_train.json ...\u001b[0m\n",
      "\u001b[32m2026-01-04 09:01:27.018\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.benchmark.pertqa\u001b[0m:\u001b[36m_load_data_from_file\u001b[0m:\u001b[36m52\u001b[0m - \u001b[1mloading HotPotQA data from /home/tl688/pitl688/selfevolve/EvoAgentX/examples/pertqa/reploge_train.json ...\u001b[0m\n",
      "\u001b[32m2026-01-04 09:01:27.087\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.benchmark.pertqa\u001b[0m:\u001b[36m_load_data_from_file\u001b[0m:\u001b[36m52\u001b[0m - \u001b[1mloading HotPotQA data from /home/tl688/pitl688/selfevolve/EvoAgentX/examples/pertqa/reploge_test.json ...\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "benchmark = PertQA(pertdata='reploge')\n",
    "workflow_graph = SequentialWorkFlowGraph.from_dict(hotpotqa_graph_data)\n",
    "agent_manager = AgentManager()\n",
    "agent_manager.add_agents_from_workflow(workflow_graph, executor_llm.config)\n",
    "\n",
    "evaluator = Evaluator(\n",
    "    llm=executor_llm, \n",
    "    agent_manager=agent_manager, \n",
    "    collate_func=collate_func, \n",
    "    num_workers=20, \n",
    "    verbose=True\n",
    ")\n",
    "\n",
    "textgrad_optimizer = TextGradOptimizer(\n",
    "    graph=workflow_graph, \n",
    "    optimize_mode=\"all\",\n",
    "    executor_llm=executor_llm, \n",
    "    optimizer_llm=optimizer_llm,\n",
    "    batch_size=3,\n",
    "    max_steps=20,\n",
    "    evaluator=evaluator,\n",
    "    eval_every_n_steps=1,\n",
    "    eval_rounds=1,\n",
    "    save_interval=None,\n",
    "    save_path=\"./\",\n",
    "    rollback=True,\n",
    "    constraints=[]\n",
    ")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "78d5904e",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# len(benchmark._fulldata)\n",
    "benchmark._train_data = benchmark._train_data[0:50]\n",
    "benchmark._dev_data = benchmark._dev_data[0:50]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "cb0d1f3d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3000"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(benchmark._test_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "d686ee20",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[32m2026-01-04 09:01:45.684\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36m__main__\u001b[0m:\u001b[36m<module>\u001b[0m:\u001b[36m1\u001b[0m - \u001b[1mEvaluating workflow on test set...\u001b[0m\n",
      "Evaluating workflow:   0%|          | 2/3000 [00:04<1:40:53,  2.02s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   0%|          | 4/3000 [00:05<41:21,  1.21it/s]  "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   0%|          | 9/3000 [00:05<13:05,  3.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   0%|          | 12/3000 [00:06<11:37,  4.28it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   0%|          | 14/3000 [00:06<10:06,  4.92it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   1%|          | 17/3000 [00:07<07:48,  6.37it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   1%|          | 19/3000 [00:07<10:51,  4.58it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   1%|          | 20/3000 [00:08<11:09,  4.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   1%|          | 21/3000 [00:08<14:58,  3.32it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   1%|          | 22/3000 [00:09<20:22,  2.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   1%|          | 23/3000 [00:09<17:35,  2.82it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   1%|          | 24/3000 [00:09<16:46,  2.96it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   1%|          | 27/3000 [00:10<13:43,  3.61it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   1%|          | 30/3000 [00:10<09:26,  5.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   1%|          | 32/3000 [00:11<08:26,  5.86it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   1%|          | 35/3000 [00:11<07:26,  6.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   1%|▏         | 38/3000 [00:12<08:12,  6.02it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   1%|▏         | 39/3000 [00:12<09:03,  5.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   1%|▏         | 41/3000 [00:12<08:45,  5.63it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   1%|▏         | 42/3000 [00:13<15:49,  3.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   1%|▏         | 43/3000 [00:13<14:43,  3.35it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   2%|▏         | 45/3000 [00:14<13:41,  3.60it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   2%|▏         | 47/3000 [00:14<09:57,  4.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   2%|▏         | 48/3000 [00:14<10:30,  4.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   2%|▏         | 52/3000 [00:15<06:49,  7.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   2%|▏         | 53/3000 [00:16<14:52,  3.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   2%|▏         | 55/3000 [00:16<12:48,  3.83it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   2%|▏         | 56/3000 [00:16<12:56,  3.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   2%|▏         | 59/3000 [00:17<08:25,  5.82it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   2%|▏         | 61/3000 [00:17<09:46,  5.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   2%|▏         | 62/3000 [00:18<11:18,  4.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   2%|▏         | 63/3000 [00:18<13:14,  3.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   2%|▏         | 64/3000 [00:18<13:26,  3.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   2%|▏         | 66/3000 [00:19<14:06,  3.47it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   2%|▏         | 69/3000 [00:19<09:13,  5.29it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   2%|▏         | 70/3000 [00:20<13:32,  3.60it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   2%|▏         | 73/3000 [00:20<10:09,  4.80it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   2%|▎         | 75/3000 [00:21<11:20,  4.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   3%|▎         | 77/3000 [00:21<11:43,  4.16it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   3%|▎         | 78/3000 [00:22<11:28,  4.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   3%|▎         | 79/3000 [00:22<11:12,  4.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   3%|▎         | 80/3000 [00:22<11:51,  4.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   3%|▎         | 81/3000 [00:22<11:41,  4.16it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   3%|▎         | 83/3000 [00:23<13:05,  3.72it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   3%|▎         | 84/3000 [00:23<14:07,  3.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   3%|▎         | 85/3000 [00:23<13:07,  3.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   3%|▎         | 86/3000 [00:24<12:31,  3.88it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   3%|▎         | 89/3000 [00:24<07:46,  6.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   3%|▎         | 90/3000 [00:24<08:48,  5.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   3%|▎         | 93/3000 [00:25<11:40,  4.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   3%|▎         | 94/3000 [00:25<11:17,  4.29it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   3%|▎         | 98/3000 [00:26<09:28,  5.11it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   3%|▎         | 99/3000 [00:26<10:19,  4.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   3%|▎         | 103/3000 [00:27<10:04,  4.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   4%|▎         | 106/3000 [00:28<12:07,  3.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   4%|▎         | 111/3000 [00:29<09:18,  5.17it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   4%|▍         | 113/3000 [00:30<09:37,  5.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   4%|▍         | 116/3000 [00:30<08:04,  5.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   4%|▍         | 118/3000 [00:30<07:19,  6.55it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   4%|▍         | 120/3000 [00:31<11:05,  4.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   4%|▍         | 121/3000 [00:32<15:56,  3.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   4%|▍         | 122/3000 [00:32<14:48,  3.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   4%|▍         | 124/3000 [00:32<11:20,  4.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   4%|▍         | 125/3000 [00:33<11:55,  4.02it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   4%|▍         | 129/3000 [00:33<08:10,  5.86it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   4%|▍         | 131/3000 [00:33<07:22,  6.48it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   4%|▍         | 133/3000 [00:34<11:49,  4.04it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   4%|▍         | 135/3000 [00:34<09:22,  5.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   5%|▍         | 136/3000 [00:34<08:34,  5.57it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   5%|▍         | 138/3000 [00:35<08:11,  5.82it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   5%|▍         | 139/3000 [00:35<08:36,  5.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   5%|▍         | 140/3000 [00:35<12:00,  3.97it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   5%|▍         | 141/3000 [00:36<18:48,  2.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   5%|▍         | 142/3000 [00:36<16:25,  2.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   5%|▍         | 145/3000 [00:37<10:26,  4.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   5%|▍         | 147/3000 [00:37<08:17,  5.74it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   5%|▍         | 149/3000 [00:37<08:15,  5.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   5%|▌         | 151/3000 [00:38<07:16,  6.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   5%|▌         | 154/3000 [00:38<08:39,  5.48it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   5%|▌         | 158/3000 [00:39<06:49,  6.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   5%|▌         | 160/3000 [00:39<08:24,  5.63it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   5%|▌         | 161/3000 [00:40<15:30,  3.05it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   5%|▌         | 162/3000 [00:41<15:09,  3.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   6%|▌         | 165/3000 [00:41<10:03,  4.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   6%|▌         | 168/3000 [00:41<06:58,  6.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   6%|▌         | 170/3000 [00:42<08:07,  5.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   6%|▌         | 172/3000 [00:42<08:07,  5.80it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   6%|▌         | 173/3000 [00:42<10:04,  4.67it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   6%|▌         | 175/3000 [00:43<09:42,  4.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   6%|▌         | 177/3000 [00:43<08:51,  5.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   6%|▌         | 178/3000 [00:43<09:30,  4.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   6%|▌         | 179/3000 [00:44<13:36,  3.46it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   6%|▌         | 180/3000 [00:44<14:22,  3.27it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   6%|▌         | 182/3000 [00:45<12:35,  3.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   6%|▌         | 184/3000 [00:45<12:10,  3.86it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   6%|▌         | 187/3000 [00:46<09:01,  5.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   6%|▋         | 188/3000 [00:46<08:23,  5.58it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   6%|▋         | 190/3000 [00:46<08:00,  5.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   6%|▋         | 191/3000 [00:46<09:19,  5.02it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   6%|▋         | 193/3000 [00:47<08:52,  5.27it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   6%|▋         | 195/3000 [00:47<09:25,  4.96it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   7%|▋         | 196/3000 [00:47<08:27,  5.52it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   7%|▋         | 197/3000 [00:48<11:30,  4.06it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   7%|▋         | 198/3000 [00:48<12:58,  3.60it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   7%|▋         | 200/3000 [00:49<14:05,  3.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   7%|▋         | 201/3000 [00:49<13:44,  3.40it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   7%|▋         | 202/3000 [00:49<13:43,  3.40it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   7%|▋         | 205/3000 [00:50<09:53,  4.71it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   7%|▋         | 207/3000 [00:50<09:55,  4.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   7%|▋         | 210/3000 [00:51<08:09,  5.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   7%|▋         | 212/3000 [00:51<06:48,  6.82it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   7%|▋         | 213/3000 [00:51<06:45,  6.87it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   7%|▋         | 216/3000 [00:51<06:26,  7.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   7%|▋         | 217/3000 [00:52<06:36,  7.02it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   7%|▋         | 218/3000 [00:52<11:10,  4.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   7%|▋         | 219/3000 [00:52<12:13,  3.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   7%|▋         | 221/3000 [00:53<11:53,  3.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   7%|▋         | 222/3000 [00:54<16:00,  2.89it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   8%|▊         | 225/3000 [00:54<11:19,  4.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   8%|▊         | 227/3000 [00:54<10:28,  4.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   8%|▊         | 228/3000 [00:55<12:47,  3.61it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   8%|▊         | 230/3000 [00:55<11:41,  3.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   8%|▊         | 236/3000 [00:56<06:43,  6.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   8%|▊         | 237/3000 [00:56<07:38,  6.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   8%|▊         | 238/3000 [00:57<09:12,  5.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   8%|▊         | 239/3000 [00:57<12:24,  3.71it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   8%|▊         | 242/3000 [00:58<10:06,  4.55it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   8%|▊         | 243/3000 [00:58<11:08,  4.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   8%|▊         | 245/3000 [00:58<09:48,  4.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   8%|▊         | 246/3000 [00:59<11:17,  4.07it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   8%|▊         | 247/3000 [00:59<13:50,  3.32it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   8%|▊         | 248/3000 [00:59<12:53,  3.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   8%|▊         | 250/3000 [01:00<10:21,  4.42it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   8%|▊         | 255/3000 [01:00<06:54,  6.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   9%|▊         | 257/3000 [01:00<06:13,  7.35it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   9%|▊         | 258/3000 [01:02<15:49,  2.89it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   9%|▊         | 260/3000 [01:02<13:37,  3.35it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   9%|▊         | 262/3000 [01:02<10:53,  4.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   9%|▉         | 264/3000 [01:03<08:59,  5.07it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   9%|▉         | 266/3000 [01:03<10:42,  4.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   9%|▉         | 270/3000 [01:04<07:18,  6.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   9%|▉         | 271/3000 [01:04<06:56,  6.55it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   9%|▉         | 272/3000 [01:04<08:24,  5.40it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   9%|▉         | 273/3000 [01:05<10:29,  4.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   9%|▉         | 275/3000 [01:05<11:32,  3.93it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   9%|▉         | 276/3000 [01:06<14:01,  3.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   9%|▉         | 280/3000 [01:06<09:38,  4.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   9%|▉         | 281/3000 [01:06<08:39,  5.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   9%|▉         | 282/3000 [01:07<10:07,  4.47it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  10%|▉         | 285/3000 [01:07<09:30,  4.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  10%|▉         | 287/3000 [01:08<06:55,  6.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  10%|▉         | 289/3000 [01:08<06:29,  6.96it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  10%|▉         | 290/3000 [01:08<09:08,  4.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  10%|▉         | 293/3000 [01:10<13:22,  3.38it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  10%|▉         | 294/3000 [01:10<11:44,  3.84it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  10%|▉         | 295/3000 [01:10<11:48,  3.82it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  10%|█         | 300/3000 [01:11<06:19,  7.11it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  10%|█         | 303/3000 [01:11<04:40,  9.61it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  10%|█         | 305/3000 [01:12<08:41,  5.17it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  10%|█         | 307/3000 [01:12<10:32,  4.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  10%|█         | 310/3000 [01:13<10:41,  4.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  10%|█         | 311/3000 [01:13<12:55,  3.47it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  10%|█         | 313/3000 [01:14<10:59,  4.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  10%|█         | 315/3000 [01:14<09:40,  4.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  11%|█         | 317/3000 [01:15<09:51,  4.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  11%|█         | 319/3000 [01:15<08:01,  5.57it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  11%|█         | 320/3000 [01:15<07:22,  6.05it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  11%|█         | 322/3000 [01:15<07:15,  6.14it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  11%|█         | 323/3000 [01:16<09:05,  4.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  11%|█         | 325/3000 [01:16<08:46,  5.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  11%|█         | 327/3000 [01:16<09:21,  4.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  11%|█         | 328/3000 [01:17<11:54,  3.74it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  11%|█         | 331/3000 [01:17<09:01,  4.93it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  11%|█         | 332/3000 [01:17<08:17,  5.36it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  11%|█         | 333/3000 [01:18<11:04,  4.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  11%|█         | 335/3000 [01:18<09:04,  4.89it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  11%|█         | 336/3000 [01:18<09:33,  4.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  11%|█▏        | 339/3000 [01:19<06:33,  6.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  11%|█▏        | 340/3000 [01:19<09:45,  4.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  11%|█▏        | 343/3000 [01:20<08:08,  5.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  12%|█▏        | 346/3000 [01:21<09:08,  4.84it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  12%|█▏        | 348/3000 [01:21<08:54,  4.96it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  12%|█▏        | 349/3000 [01:21<09:19,  4.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  12%|█▏        | 350/3000 [01:21<09:29,  4.66it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  12%|█▏        | 352/3000 [01:22<11:26,  3.86it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  12%|█▏        | 353/3000 [01:23<16:34,  2.66it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  12%|█▏        | 356/3000 [01:24<14:14,  3.09it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  12%|█▏        | 359/3000 [01:24<08:11,  5.38it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  12%|█▏        | 360/3000 [01:24<08:13,  5.35it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  12%|█▏        | 362/3000 [01:24<06:57,  6.32it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  12%|█▏        | 364/3000 [01:25<09:29,  4.63it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  12%|█▏        | 365/3000 [01:25<11:13,  3.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  12%|█▏        | 366/3000 [01:26<11:46,  3.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  12%|█▏        | 370/3000 [01:27<12:42,  3.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  12%|█▏        | 372/3000 [01:27<10:34,  4.14it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  12%|█▏        | 373/3000 [01:27<10:11,  4.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  12%|█▎        | 375/3000 [01:28<11:22,  3.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  13%|█▎        | 377/3000 [01:29<10:46,  4.06it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  13%|█▎        | 382/3000 [01:29<05:15,  8.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  13%|█▎        | 385/3000 [01:30<07:03,  6.17it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  13%|█▎        | 387/3000 [01:30<08:32,  5.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  13%|█▎        | 389/3000 [01:31<11:17,  3.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  13%|█▎        | 390/3000 [01:31<14:12,  3.06it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  13%|█▎        | 392/3000 [01:32<11:28,  3.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  13%|█▎        | 394/3000 [01:32<07:47,  5.58it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  13%|█▎        | 398/3000 [01:32<06:14,  6.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  13%|█▎        | 400/3000 [01:33<09:16,  4.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  13%|█▎        | 403/3000 [01:34<09:20,  4.63it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  14%|█▎        | 405/3000 [01:34<06:58,  6.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  14%|█▎        | 407/3000 [01:34<06:07,  7.06it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  14%|█▎        | 408/3000 [01:35<11:20,  3.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  14%|█▎        | 409/3000 [01:35<11:08,  3.88it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  14%|█▎        | 410/3000 [01:35<11:00,  3.92it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  14%|█▎        | 412/3000 [01:36<10:34,  4.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  14%|█▍        | 413/3000 [01:36<13:07,  3.28it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  14%|█▍        | 418/3000 [01:37<06:57,  6.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  14%|█▍        | 419/3000 [01:38<09:40,  4.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  14%|█▍        | 423/3000 [01:38<06:28,  6.63it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  14%|█▍        | 424/3000 [01:38<07:46,  5.52it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  14%|█▍        | 426/3000 [01:39<09:10,  4.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  14%|█▍        | 427/3000 [01:39<12:39,  3.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  14%|█▍        | 431/3000 [01:40<07:20,  5.83it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  14%|█▍        | 432/3000 [01:40<06:57,  6.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  14%|█▍        | 435/3000 [01:40<06:10,  6.93it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  15%|█▍        | 436/3000 [01:41<07:35,  5.63it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  15%|█▍        | 440/3000 [01:41<06:31,  6.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  15%|█▍        | 441/3000 [01:42<12:25,  3.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  15%|█▍        | 442/3000 [01:43<15:48,  2.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  15%|█▍        | 443/3000 [01:43<15:17,  2.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  15%|█▍        | 445/3000 [01:44<12:46,  3.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metricsmetrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      " {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  15%|█▌        | 450/3000 [01:44<06:44,  6.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  15%|█▌        | 451/3000 [01:45<10:46,  3.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  15%|█▌        | 456/3000 [01:45<07:15,  5.84it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  15%|█▌        | 458/3000 [01:46<06:38,  6.38it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  15%|█▌        | 460/3000 [01:46<09:41,  4.37it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  15%|█▌        | 461/3000 [01:46<09:06,  4.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  15%|█▌        | 462/3000 [01:47<14:48,  2.86it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  15%|█▌        | 464/3000 [01:48<14:08,  2.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  16%|█▌        | 467/3000 [01:48<07:58,  5.29it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  16%|█▌        | 469/3000 [01:48<06:18,  6.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  16%|█▌        | 471/3000 [01:49<07:37,  5.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  16%|█▌        | 472/3000 [01:49<08:17,  5.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  16%|█▌        | 474/3000 [01:50<08:40,  4.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  16%|█▌        | 477/3000 [01:50<05:43,  7.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  16%|█▌        | 479/3000 [01:50<08:47,  4.77it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  16%|█▌        | 480/3000 [01:51<11:54,  3.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  16%|█▌        | 481/3000 [01:52<14:24,  2.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  16%|█▌        | 484/3000 [01:52<09:41,  4.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  16%|█▌        | 487/3000 [01:53<08:52,  4.72it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  16%|█▋        | 488/3000 [01:53<08:24,  4.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  16%|█▋        | 490/3000 [01:53<08:56,  4.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  16%|█▋        | 491/3000 [01:53<09:04,  4.61it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  16%|█▋        | 493/3000 [01:54<07:53,  5.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  16%|█▋        | 495/3000 [01:54<07:04,  5.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  17%|█▋        | 497/3000 [01:54<06:26,  6.48it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  17%|█▋        | 498/3000 [01:55<07:01,  5.93it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  17%|█▋        | 500/3000 [01:55<07:19,  5.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  17%|█▋        | 501/3000 [01:55<08:20,  4.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  17%|█▋        | 503/3000 [01:56<10:21,  4.02it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  17%|█▋        | 505/3000 [01:56<08:23,  4.96it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  17%|█▋        | 507/3000 [01:57<08:11,  5.07it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  17%|█▋        | 508/3000 [01:57<11:44,  3.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  17%|█▋        | 510/3000 [01:58<17:02,  2.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  17%|█▋        | 513/3000 [01:59<10:26,  3.97it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  17%|█▋        | 514/3000 [01:59<09:15,  4.48it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  17%|█▋        | 517/3000 [01:59<07:56,  5.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  17%|█▋        | 520/3000 [01:59<05:09,  8.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  17%|█▋        | 524/3000 [02:00<05:33,  7.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  18%|█▊        | 528/3000 [02:01<08:33,  4.82it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  18%|█▊        | 529/3000 [02:02<10:20,  3.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  18%|█▊        | 530/3000 [02:02<12:54,  3.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  18%|█▊        | 531/3000 [02:03<12:26,  3.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  18%|█▊        | 532/3000 [02:03<11:31,  3.57it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  18%|█▊        | 536/3000 [02:03<06:55,  5.93it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  18%|█▊        | 537/3000 [02:04<07:48,  5.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  18%|█▊        | 539/3000 [02:04<06:36,  6.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  18%|█▊        | 543/3000 [02:04<04:37,  8.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  18%|█▊        | 545/3000 [02:05<08:50,  4.63it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  18%|█▊        | 546/3000 [02:05<10:15,  3.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  18%|█▊        | 548/3000 [02:06<11:46,  3.47it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  18%|█▊        | 549/3000 [02:07<12:57,  3.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  18%|█▊        | 551/3000 [02:07<09:32,  4.28it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  18%|█▊        | 554/3000 [02:07<08:02,  5.07it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  18%|█▊        | 555/3000 [02:08<09:10,  4.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  19%|█▊        | 558/3000 [02:08<07:15,  5.61it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  19%|█▊        | 560/3000 [02:08<05:21,  7.58it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  19%|█▊        | 562/3000 [02:09<06:57,  5.84it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  19%|█▉        | 563/3000 [02:09<06:51,  5.92it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  19%|█▉        | 564/3000 [02:09<09:23,  4.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  19%|█▉        | 566/3000 [02:10<12:47,  3.17it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  19%|█▉        | 567/3000 [02:11<12:00,  3.38it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  19%|█▉        | 568/3000 [02:11<11:25,  3.55it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  19%|█▉        | 569/3000 [02:11<10:48,  3.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  19%|█▉        | 572/3000 [02:11<08:04,  5.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  19%|█▉        | 575/3000 [02:12<07:34,  5.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  19%|█▉        | 576/3000 [02:12<08:31,  4.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  19%|█▉        | 578/3000 [02:13<07:42,  5.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  19%|█▉        | 580/3000 [02:13<08:36,  4.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  19%|█▉        | 581/3000 [02:13<09:41,  4.16it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  20%|█▉        | 585/3000 [02:15<13:14,  3.04it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  20%|█▉        | 590/3000 [02:16<08:33,  4.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  20%|█▉        | 592/3000 [02:16<07:07,  5.63it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  20%|█▉        | 594/3000 [02:16<08:15,  4.86it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  20%|█▉        | 596/3000 [02:17<07:41,  5.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  20%|█▉        | 597/3000 [02:17<10:15,  3.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  20%|█▉        | 598/3000 [02:17<09:51,  4.06it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metricsmetrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      " {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  20%|██        | 602/3000 [02:18<06:39,  6.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  20%|██        | 603/3000 [02:18<07:08,  5.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  20%|██        | 605/3000 [02:19<10:37,  3.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  20%|██        | 606/3000 [02:19<11:42,  3.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  20%|██        | 607/3000 [02:20<15:25,  2.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  20%|██        | 608/3000 [02:20<14:55,  2.67it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  20%|██        | 611/3000 [02:21<09:32,  4.17it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  20%|██        | 612/3000 [02:21<10:05,  3.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  20%|██        | 614/3000 [02:22<09:49,  4.05it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  20%|██        | 615/3000 [02:22<08:41,  4.57it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  21%|██        | 618/3000 [02:22<06:50,  5.80it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  21%|██        | 619/3000 [02:22<06:28,  6.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  21%|██        | 620/3000 [02:23<07:39,  5.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  21%|██        | 623/3000 [02:23<06:55,  5.71it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  21%|██        | 624/3000 [02:24<11:22,  3.48it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  21%|██        | 625/3000 [02:24<11:06,  3.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  21%|██        | 626/3000 [02:24<10:48,  3.66it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  21%|██        | 629/3000 [02:25<09:47,  4.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  21%|██        | 630/3000 [02:25<08:34,  4.60it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  21%|██        | 632/3000 [02:26<09:01,  4.37it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  21%|██        | 634/3000 [02:26<10:23,  3.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  21%|██        | 636/3000 [02:27<08:14,  4.78it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  21%|██▏       | 638/3000 [02:27<07:48,  5.04it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  21%|██▏       | 641/3000 [02:28<09:18,  4.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  21%|██▏       | 644/3000 [02:28<07:52,  4.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  22%|██▏       | 646/3000 [02:29<06:29,  6.05it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  22%|██▏       | 647/3000 [02:29<06:00,  6.52it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  22%|██▏       | 648/3000 [02:29<08:14,  4.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  22%|██▏       | 649/3000 [02:30<11:06,  3.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  22%|██▏       | 650/3000 [02:30<11:17,  3.47it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  22%|██▏       | 651/3000 [02:30<10:37,  3.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  22%|██▏       | 652/3000 [02:31<12:11,  3.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  22%|██▏       | 653/3000 [02:31<11:36,  3.37it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  22%|██▏       | 654/3000 [02:31<11:20,  3.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  22%|██▏       | 656/3000 [02:31<09:43,  4.02it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  22%|██▏       | 658/3000 [02:32<08:12,  4.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  22%|██▏       | 659/3000 [02:32<10:07,  3.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  22%|██▏       | 660/3000 [02:32<10:32,  3.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  22%|██▏       | 662/3000 [02:33<08:55,  4.36it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  22%|██▏       | 664/3000 [02:33<08:41,  4.48it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  22%|██▏       | 666/3000 [02:34<07:24,  5.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  22%|██▏       | 668/3000 [02:34<07:07,  5.46it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  22%|██▏       | 669/3000 [02:34<07:34,  5.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  22%|██▏       | 671/3000 [02:35<06:54,  5.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  22%|██▏       | 673/3000 [02:35<07:08,  5.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  22%|██▎       | 675/3000 [02:36<10:11,  3.80it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  23%|██▎       | 676/3000 [02:36<11:48,  3.28it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  23%|██▎       | 681/3000 [02:37<06:38,  5.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  23%|██▎       | 683/3000 [02:37<05:37,  6.86it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  23%|██▎       | 685/3000 [02:38<07:35,  5.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  23%|██▎       | 686/3000 [02:38<09:59,  3.86it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  23%|██▎       | 688/3000 [02:38<07:29,  5.14it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  23%|██▎       | 689/3000 [02:39<09:12,  4.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  23%|██▎       | 691/3000 [02:39<07:49,  4.92it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  23%|██▎       | 693/3000 [02:40<09:12,  4.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  23%|██▎       | 695/3000 [02:40<07:47,  4.93it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  23%|██▎       | 696/3000 [02:41<12:44,  3.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  23%|██▎       | 699/3000 [02:41<11:37,  3.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  23%|██▎       | 701/3000 [02:42<09:16,  4.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  23%|██▎       | 702/3000 [02:42<10:41,  3.58it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  24%|██▎       | 707/3000 [02:43<06:25,  5.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  24%|██▎       | 708/3000 [02:43<06:14,  6.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  24%|██▎       | 709/3000 [02:43<08:51,  4.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  24%|██▎       | 712/3000 [02:44<07:57,  4.80it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  24%|██▍       | 715/3000 [02:44<07:50,  4.86it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  24%|██▍       | 716/3000 [02:45<07:24,  5.14it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  24%|██▍       | 719/3000 [02:45<08:21,  4.55it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  24%|██▍       | 720/3000 [02:46<07:51,  4.83it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  24%|██▍       | 721/3000 [02:46<09:31,  3.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  24%|██▍       | 723/3000 [02:46<09:13,  4.11it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  24%|██▍       | 724/3000 [02:47<09:12,  4.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  24%|██▍       | 726/3000 [02:47<07:12,  5.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  24%|██▍       | 729/3000 [02:47<06:07,  6.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  24%|██▍       | 731/3000 [02:48<05:57,  6.35it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  24%|██▍       | 734/3000 [02:48<07:08,  5.29it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  25%|██▍       | 736/3000 [02:49<07:04,  5.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  25%|██▍       | 738/3000 [02:50<09:52,  3.82it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  25%|██▍       | 741/3000 [02:51<11:49,  3.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  25%|██▍       | 743/3000 [02:51<08:45,  4.29it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  25%|██▍       | 745/3000 [02:51<07:41,  4.89it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  25%|██▍       | 746/3000 [02:52<08:33,  4.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  25%|██▍       | 749/3000 [02:52<06:53,  5.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  25%|██▌       | 752/3000 [02:52<05:53,  6.36it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  25%|██▌       | 755/3000 [02:53<08:03,  4.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  25%|██▌       | 757/3000 [02:53<06:04,  6.16it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  25%|██▌       | 760/3000 [02:55<09:14,  4.04it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  25%|██▌       | 762/3000 [02:55<07:23,  5.04it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  26%|██▌       | 765/3000 [02:55<05:14,  7.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  26%|██▌       | 768/3000 [02:56<09:48,  3.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  26%|██▌       | 769/3000 [02:57<09:03,  4.11it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  26%|██▌       | 771/3000 [02:57<10:52,  3.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  26%|██▌       | 772/3000 [02:57<08:55,  4.16it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  26%|██▌       | 773/3000 [02:58<15:38,  2.37it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  26%|██▌       | 776/3000 [02:59<08:54,  4.16it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  26%|██▌       | 781/3000 [02:59<04:09,  8.88it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  26%|██▌       | 783/3000 [03:00<05:58,  6.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  26%|██▌       | 786/3000 [03:00<07:09,  5.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  26%|██▋       | 788/3000 [03:00<05:32,  6.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  26%|██▋       | 791/3000 [03:01<06:30,  5.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  26%|██▋       | 792/3000 [03:02<10:23,  3.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  26%|██▋       | 793/3000 [03:02<10:15,  3.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  26%|██▋       | 794/3000 [03:02<11:26,  3.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  27%|██▋       | 798/3000 [03:03<07:40,  4.78it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  27%|██▋       | 799/3000 [03:03<07:24,  4.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  27%|██▋       | 801/3000 [03:04<06:29,  5.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  27%|██▋       | 802/3000 [03:04<12:22,  2.96it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  27%|██▋       | 804/3000 [03:05<09:48,  3.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  27%|██▋       | 805/3000 [03:05<10:18,  3.55it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  27%|██▋       | 806/3000 [03:06<11:36,  3.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  27%|██▋       | 809/3000 [03:06<10:45,  3.40it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  27%|██▋       | 811/3000 [03:07<08:15,  4.42it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  27%|██▋       | 812/3000 [03:07<10:43,  3.40it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  27%|██▋       | 814/3000 [03:08<09:04,  4.02it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  27%|██▋       | 816/3000 [03:08<06:02,  6.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  27%|██▋       | 818/3000 [03:08<06:59,  5.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  27%|██▋       | 819/3000 [03:09<09:30,  3.83it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  27%|██▋       | 820/3000 [03:09<10:17,  3.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  27%|██▋       | 822/3000 [03:09<08:18,  4.37it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  27%|██▋       | 824/3000 [03:10<07:54,  4.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  28%|██▊       | 825/3000 [03:10<09:50,  3.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  28%|██▊       | 827/3000 [03:10<07:49,  4.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  28%|██▊       | 828/3000 [03:11<13:12,  2.74it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  28%|██▊       | 831/3000 [03:12<08:48,  4.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  28%|██▊       | 832/3000 [03:12<09:29,  3.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  28%|██▊       | 834/3000 [03:13<08:30,  4.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  28%|██▊       | 839/3000 [03:13<05:45,  6.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  28%|██▊       | 840/3000 [03:13<05:51,  6.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  28%|██▊       | 842/3000 [03:14<06:08,  5.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  28%|██▊       | 845/3000 [03:14<05:02,  7.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  28%|██▊       | 847/3000 [03:15<07:30,  4.78it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  28%|██▊       | 849/3000 [03:16<09:26,  3.80it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  28%|██▊       | 851/3000 [03:16<08:28,  4.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  28%|██▊       | 852/3000 [03:16<08:33,  4.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  28%|██▊       | 854/3000 [03:17<08:18,  4.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  29%|██▊       | 856/3000 [03:17<05:52,  6.09it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  29%|██▊       | 859/3000 [03:18<06:50,  5.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  29%|██▉       | 864/3000 [03:19<05:35,  6.36it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  29%|██▉       | 866/3000 [03:19<05:11,  6.86it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  29%|██▉       | 868/3000 [03:19<05:34,  6.38it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  29%|██▉       | 870/3000 [03:20<08:01,  4.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  29%|██▉       | 872/3000 [03:21<09:16,  3.82it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  29%|██▉       | 874/3000 [03:21<08:22,  4.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  29%|██▉       | 876/3000 [03:22<09:03,  3.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  29%|██▉       | 880/3000 [03:22<06:51,  5.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  29%|██▉       | 882/3000 [03:23<06:06,  5.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  30%|██▉       | 885/3000 [03:23<05:02,  7.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  30%|██▉       | 887/3000 [03:23<04:10,  8.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  30%|██▉       | 889/3000 [03:24<07:58,  4.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  30%|██▉       | 890/3000 [03:25<09:45,  3.60it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  30%|██▉       | 892/3000 [03:25<08:12,  4.28it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  30%|██▉       | 893/3000 [03:25<08:57,  3.92it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  30%|██▉       | 895/3000 [03:26<09:26,  3.72it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  30%|██▉       | 897/3000 [03:26<06:19,  5.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  30%|██▉       | 899/3000 [03:26<06:38,  5.27it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  30%|███       | 901/3000 [03:27<06:14,  5.60it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  30%|███       | 902/3000 [03:27<05:53,  5.93it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  30%|███       | 905/3000 [03:27<05:42,  6.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  30%|███       | 906/3000 [03:28<05:59,  5.83it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  30%|███       | 908/3000 [03:28<09:30,  3.67it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  30%|███       | 909/3000 [03:29<09:36,  3.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  30%|███       | 910/3000 [03:29<09:06,  3.83it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  30%|███       | 911/3000 [03:30<14:11,  2.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  30%|███       | 914/3000 [03:30<09:17,  3.74it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  30%|███       | 915/3000 [03:30<08:44,  3.97it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  31%|███       | 918/3000 [03:31<07:43,  4.49it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  31%|███       | 920/3000 [03:31<06:27,  5.36it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  31%|███       | 925/3000 [03:32<03:48,  9.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  31%|███       | 927/3000 [03:32<05:45,  6.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  31%|███       | 928/3000 [03:33<06:35,  5.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  31%|███       | 929/3000 [03:33<09:37,  3.58it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  31%|███       | 931/3000 [03:34<08:40,  3.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  31%|███       | 933/3000 [03:34<09:27,  3.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  31%|███       | 936/3000 [03:35<05:31,  6.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  31%|███▏      | 938/3000 [03:35<04:22,  7.84it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  31%|███▏      | 943/3000 [03:35<03:22, 10.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  32%|███▏      | 945/3000 [03:35<03:44,  9.14it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  32%|███▏      | 947/3000 [03:36<06:04,  5.63it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  32%|███▏      | 948/3000 [03:36<06:04,  5.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  32%|███▏      | 950/3000 [03:37<07:37,  4.48it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  32%|███▏      | 952/3000 [03:38<08:44,  3.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  32%|███▏      | 953/3000 [03:38<08:42,  3.92it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  32%|███▏      | 954/3000 [03:38<08:20,  4.09it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  32%|███▏      | 955/3000 [03:38<09:56,  3.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  32%|███▏      | 958/3000 [03:39<07:27,  4.57it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  32%|███▏      | 960/3000 [03:39<07:54,  4.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  32%|███▏      | 961/3000 [03:40<08:50,  3.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  32%|███▏      | 964/3000 [03:40<06:25,  5.27it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  32%|███▏      | 967/3000 [03:41<05:34,  6.07it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  32%|███▏      | 969/3000 [03:41<05:49,  5.82it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  32%|███▏      | 970/3000 [03:41<05:49,  5.82it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  32%|███▏      | 972/3000 [03:42<07:16,  4.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  33%|███▎      | 977/3000 [03:42<04:29,  7.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  33%|███▎      | 979/3000 [03:43<08:59,  3.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  33%|███▎      | 981/3000 [03:44<07:43,  4.35it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  33%|███▎      | 983/3000 [03:44<06:33,  5.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  33%|███▎      | 985/3000 [03:45<08:05,  4.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  33%|███▎      | 986/3000 [03:45<07:27,  4.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  33%|███▎      | 989/3000 [03:45<06:17,  5.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  33%|███▎      | 990/3000 [03:46<07:12,  4.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  33%|███▎      | 992/3000 [03:47<09:09,  3.66it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  33%|███▎      | 994/3000 [03:47<07:21,  4.55it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  33%|███▎      | 996/3000 [03:47<06:11,  5.40it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  33%|███▎      | 999/3000 [03:47<05:20,  6.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  33%|███▎      | 1000/3000 [03:48<05:58,  5.58it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  33%|███▎      | 1001/3000 [03:48<07:01,  4.74it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  33%|███▎      | 1002/3000 [03:48<06:59,  4.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  33%|███▎      | 1003/3000 [03:49<08:32,  3.89it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  34%|███▎      | 1005/3000 [03:49<07:56,  4.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  34%|███▎      | 1006/3000 [03:49<08:01,  4.14it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  34%|███▎      | 1008/3000 [03:50<07:59,  4.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  34%|███▎      | 1009/3000 [03:50<09:26,  3.52it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  34%|███▎      | 1012/3000 [03:51<06:16,  5.28it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  34%|███▍      | 1013/3000 [03:51<06:16,  5.27it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  34%|███▍      | 1014/3000 [03:51<06:48,  4.86it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  34%|███▍      | 1016/3000 [03:51<05:59,  5.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  34%|███▍      | 1018/3000 [03:52<06:41,  4.93it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  34%|███▍      | 1022/3000 [03:52<03:32,  9.29it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  34%|███▍      | 1024/3000 [03:53<05:31,  5.96it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  34%|███▍      | 1025/3000 [03:53<05:16,  6.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  34%|███▍      | 1026/3000 [03:53<08:51,  3.71it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  34%|███▍      | 1028/3000 [03:54<08:52,  3.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  34%|███▍      | 1030/3000 [03:54<06:24,  5.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  34%|███▍      | 1031/3000 [03:54<06:46,  4.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  34%|███▍      | 1033/3000 [03:55<06:05,  5.38it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  34%|███▍      | 1035/3000 [03:55<07:55,  4.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  35%|███▍      | 1038/3000 [03:56<06:33,  4.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  35%|███▍      | 1041/3000 [03:56<04:30,  7.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metricsmetrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      " {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  35%|███▍      | 1042/3000 [03:56<04:24,  7.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  35%|███▍      | 1044/3000 [03:57<06:07,  5.32it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  35%|███▍      | 1045/3000 [03:58<12:09,  2.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  35%|███▍      | 1049/3000 [03:58<07:12,  4.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  35%|███▌      | 1052/3000 [03:59<05:09,  6.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  35%|███▌      | 1054/3000 [03:59<07:01,  4.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  35%|███▌      | 1055/3000 [03:59<06:12,  5.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  35%|███▌      | 1056/3000 [04:00<07:28,  4.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  35%|███▌      | 1061/3000 [04:01<05:28,  5.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  35%|███▌      | 1063/3000 [04:01<04:12,  7.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  36%|███▌      | 1065/3000 [04:02<09:55,  3.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  36%|███▌      | 1066/3000 [04:02<09:25,  3.42it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  36%|███▌      | 1068/3000 [04:03<08:08,  3.96it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  36%|███▌      | 1071/3000 [04:03<06:02,  5.32it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  36%|███▌      | 1073/3000 [04:03<05:30,  5.83it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  36%|███▌      | 1074/3000 [04:04<10:39,  3.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  36%|███▌      | 1078/3000 [04:05<06:32,  4.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  36%|███▌      | 1080/3000 [04:05<05:12,  6.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  36%|███▌      | 1083/3000 [04:05<05:28,  5.84it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  36%|███▌      | 1084/3000 [04:06<05:02,  6.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  36%|███▌      | 1085/3000 [04:06<08:24,  3.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  36%|███▌      | 1087/3000 [04:06<06:20,  5.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  36%|███▋      | 1089/3000 [04:07<05:42,  5.58it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  36%|███▋      | 1090/3000 [04:07<06:13,  5.11it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  36%|███▋      | 1093/3000 [04:08<06:19,  5.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  36%|███▋      | 1094/3000 [04:08<09:39,  3.29it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  36%|███▋      | 1095/3000 [04:09<09:50,  3.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  37%|███▋      | 1098/3000 [04:09<06:40,  4.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  37%|███▋      | 1099/3000 [04:09<06:56,  4.57it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  37%|███▋      | 1100/3000 [04:10<06:57,  4.55it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  37%|███▋      | 1105/3000 [04:10<04:25,  7.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  37%|███▋      | 1107/3000 [04:10<04:24,  7.16it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  37%|███▋      | 1109/3000 [04:11<05:04,  6.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  37%|███▋      | 1111/3000 [04:11<05:05,  6.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  37%|███▋      | 1112/3000 [04:12<08:56,  3.52it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  37%|███▋      | 1113/3000 [04:12<10:48,  2.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  37%|███▋      | 1115/3000 [04:13<09:26,  3.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  37%|███▋      | 1119/3000 [04:13<06:27,  4.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  37%|███▋      | 1120/3000 [04:13<06:05,  5.14it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  37%|███▋      | 1121/3000 [04:14<06:30,  4.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  38%|███▊      | 1125/3000 [04:14<04:34,  6.83it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  38%|███▊      | 1126/3000 [04:14<04:48,  6.49it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  38%|███▊      | 1127/3000 [04:15<07:26,  4.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  38%|███▊      | 1129/3000 [04:15<06:17,  4.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  38%|███▊      | 1131/3000 [04:16<07:02,  4.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  38%|███▊      | 1132/3000 [04:16<06:41,  4.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  38%|███▊      | 1134/3000 [04:16<07:10,  4.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  38%|███▊      | 1135/3000 [04:17<07:47,  3.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  38%|███▊      | 1136/3000 [04:17<08:08,  3.82it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  38%|███▊      | 1138/3000 [04:18<08:09,  3.80it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  38%|███▊      | 1140/3000 [04:18<07:08,  4.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  38%|███▊      | 1143/3000 [04:18<05:04,  6.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  38%|███▊      | 1144/3000 [04:19<05:02,  6.14it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  38%|███▊      | 1145/3000 [04:19<07:11,  4.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  38%|███▊      | 1146/3000 [04:19<07:35,  4.07it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  38%|███▊      | 1147/3000 [04:20<08:58,  3.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  38%|███▊      | 1149/3000 [04:20<06:23,  4.83it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  38%|███▊      | 1152/3000 [04:21<06:33,  4.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  38%|███▊      | 1154/3000 [04:21<04:54,  6.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  38%|███▊      | 1155/3000 [04:21<05:45,  5.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  39%|███▊      | 1158/3000 [04:22<06:44,  4.55it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  39%|███▊      | 1160/3000 [04:22<05:57,  5.14it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  39%|███▊      | 1162/3000 [04:23<04:32,  6.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  39%|███▉      | 1165/3000 [04:23<05:41,  5.37it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  39%|███▉      | 1168/3000 [04:24<04:42,  6.49it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  39%|███▉      | 1169/3000 [04:24<05:52,  5.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  39%|███▉      | 1170/3000 [04:24<07:03,  4.32it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  39%|███▉      | 1171/3000 [04:25<09:51,  3.09it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  39%|███▉      | 1174/3000 [04:25<06:00,  5.06it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  39%|███▉      | 1175/3000 [04:26<07:34,  4.02it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  39%|███▉      | 1178/3000 [04:26<05:22,  5.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  39%|███▉      | 1181/3000 [04:27<07:06,  4.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  39%|███▉      | 1184/3000 [04:27<04:45,  6.36it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  40%|███▉      | 1187/3000 [04:28<05:05,  5.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  40%|███▉      | 1188/3000 [04:28<08:10,  3.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  40%|███▉      | 1190/3000 [04:29<07:37,  3.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  40%|███▉      | 1191/3000 [04:29<09:51,  3.06it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  40%|███▉      | 1195/3000 [04:30<05:57,  5.05it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  40%|███▉      | 1198/3000 [04:31<05:37,  5.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  40%|████      | 1200/3000 [04:31<04:59,  6.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  40%|████      | 1202/3000 [04:31<04:52,  6.14it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  40%|████      | 1205/3000 [04:32<03:53,  7.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  40%|████      | 1208/3000 [04:32<05:34,  5.35it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  40%|████      | 1210/3000 [04:33<05:21,  5.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  40%|████      | 1211/3000 [04:34<10:21,  2.88it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  40%|████      | 1214/3000 [04:34<07:33,  3.93it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  41%|████      | 1216/3000 [04:35<06:04,  4.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  41%|████      | 1218/3000 [04:35<06:17,  4.72it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  41%|████      | 1224/3000 [04:36<03:49,  7.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  41%|████      | 1226/3000 [04:36<04:44,  6.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  41%|████      | 1229/3000 [04:37<04:38,  6.36it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  41%|████      | 1230/3000 [04:37<05:28,  5.38it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  41%|████      | 1232/3000 [04:38<10:23,  2.84it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  41%|████      | 1235/3000 [04:39<07:18,  4.02it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  41%|████▏     | 1238/3000 [04:39<04:55,  5.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  41%|████▏     | 1241/3000 [04:39<03:28,  8.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  41%|████▏     | 1244/3000 [04:40<04:36,  6.36it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  42%|████▏     | 1246/3000 [04:41<06:30,  4.49it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  42%|████▏     | 1247/3000 [04:41<06:29,  4.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  42%|████▏     | 1248/3000 [04:41<06:22,  4.58it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  42%|████▏     | 1250/3000 [04:42<06:38,  4.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  42%|████▏     | 1251/3000 [04:42<08:49,  3.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  42%|████▏     | 1253/3000 [04:43<08:00,  3.63it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  42%|████▏     | 1256/3000 [04:43<06:42,  4.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  42%|████▏     | 1262/3000 [04:44<03:58,  7.28it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  42%|████▏     | 1263/3000 [04:44<04:06,  7.06it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  42%|████▏     | 1264/3000 [04:45<05:46,  5.02it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  42%|████▏     | 1266/3000 [04:45<05:44,  5.04it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  42%|████▏     | 1268/3000 [04:45<04:38,  6.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  42%|████▏     | 1270/3000 [04:46<05:42,  5.05it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  42%|████▏     | 1271/3000 [04:46<07:21,  3.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  42%|████▏     | 1273/3000 [04:47<06:00,  4.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  42%|████▏     | 1274/3000 [04:47<06:16,  4.58it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  43%|████▎     | 1277/3000 [04:47<05:12,  5.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  43%|████▎     | 1278/3000 [04:48<07:01,  4.09it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  43%|████▎     | 1282/3000 [04:48<05:21,  5.35it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  43%|████▎     | 1285/3000 [04:49<03:26,  8.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  43%|████▎     | 1287/3000 [04:49<04:33,  6.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  43%|████▎     | 1290/3000 [04:49<03:34,  7.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  43%|████▎     | 1292/3000 [04:50<06:58,  4.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  43%|████▎     | 1297/3000 [04:51<05:54,  4.80it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  43%|████▎     | 1301/3000 [04:52<04:34,  6.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  44%|████▎     | 1305/3000 [04:53<04:31,  6.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  44%|████▎     | 1307/3000 [04:53<04:54,  5.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  44%|████▎     | 1309/3000 [04:53<05:32,  5.09it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  44%|████▎     | 1310/3000 [04:54<07:14,  3.89it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  44%|████▍     | 1313/3000 [04:55<05:50,  4.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  44%|████▍     | 1315/3000 [04:55<06:57,  4.04it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  44%|████▍     | 1316/3000 [04:55<06:11,  4.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  44%|████▍     | 1318/3000 [04:56<08:17,  3.38it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  44%|████▍     | 1322/3000 [04:56<03:39,  7.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  44%|████▍     | 1324/3000 [04:57<04:26,  6.29it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  44%|████▍     | 1327/3000 [04:57<04:49,  5.78it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  44%|████▍     | 1328/3000 [04:58<05:03,  5.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  44%|████▍     | 1330/3000 [04:58<06:06,  4.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  44%|████▍     | 1331/3000 [04:59<07:25,  3.74it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  44%|████▍     | 1332/3000 [04:59<06:59,  3.97it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  44%|████▍     | 1334/3000 [05:00<08:57,  3.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  45%|████▍     | 1336/3000 [05:00<06:01,  4.61it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  45%|████▍     | 1339/3000 [05:00<05:28,  5.06it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  45%|████▍     | 1341/3000 [05:01<04:24,  6.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  45%|████▍     | 1345/3000 [05:01<04:51,  5.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  45%|████▍     | 1346/3000 [05:02<06:05,  4.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  45%|████▍     | 1348/3000 [05:03<09:39,  2.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  45%|████▌     | 1351/3000 [05:04<06:58,  3.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  45%|████▌     | 1353/3000 [05:04<05:19,  5.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  45%|████▌     | 1355/3000 [05:04<04:17,  6.40it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  45%|████▌     | 1356/3000 [05:04<05:12,  5.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  45%|████▌     | 1357/3000 [05:04<05:20,  5.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  45%|████▌     | 1360/3000 [05:05<04:09,  6.58it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  45%|████▌     | 1362/3000 [05:05<03:45,  7.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  46%|████▌     | 1365/3000 [05:06<04:50,  5.63it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  46%|████▌     | 1367/3000 [05:06<06:04,  4.48it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  46%|████▌     | 1368/3000 [05:07<07:28,  3.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  46%|████▌     | 1370/3000 [05:07<06:17,  4.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  46%|████▌     | 1371/3000 [05:08<06:14,  4.35it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  46%|████▌     | 1374/3000 [05:08<04:51,  5.57it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  46%|████▌     | 1377/3000 [05:09<04:29,  6.02it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  46%|████▌     | 1379/3000 [05:09<03:46,  7.14it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  46%|████▌     | 1385/3000 [05:09<02:32, 10.61it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  46%|████▋     | 1388/3000 [05:10<05:26,  4.93it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  46%|████▋     | 1390/3000 [05:11<04:26,  6.05it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  46%|████▋     | 1391/3000 [05:11<06:34,  4.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  46%|████▋     | 1392/3000 [05:12<10:41,  2.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  47%|████▋     | 1397/3000 [05:12<05:00,  5.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  47%|████▋     | 1399/3000 [05:13<04:07,  6.46it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  47%|████▋     | 1401/3000 [05:13<04:50,  5.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  47%|████▋     | 1403/3000 [05:14<05:02,  5.28it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  47%|████▋     | 1405/3000 [05:14<04:15,  6.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  47%|████▋     | 1407/3000 [05:14<04:38,  5.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  47%|████▋     | 1410/3000 [05:15<04:01,  6.58it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  47%|████▋     | 1411/3000 [05:15<03:45,  7.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  47%|████▋     | 1413/3000 [05:15<05:45,  4.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  47%|████▋     | 1414/3000 [05:16<06:15,  4.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  47%|████▋     | 1416/3000 [05:16<05:53,  4.48it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  47%|████▋     | 1419/3000 [05:16<04:21,  6.05it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  47%|████▋     | 1421/3000 [05:17<04:08,  6.35it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  47%|████▋     | 1424/3000 [05:18<05:39,  4.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  48%|████▊     | 1426/3000 [05:18<04:32,  5.77it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  48%|████▊     | 1428/3000 [05:18<04:26,  5.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  48%|████▊     | 1431/3000 [05:19<03:51,  6.78it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  48%|████▊     | 1433/3000 [05:19<05:32,  4.71it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  48%|████▊     | 1435/3000 [05:20<05:21,  4.87it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  48%|████▊     | 1437/3000 [05:21<08:00,  3.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  48%|████▊     | 1440/3000 [05:21<06:26,  4.04it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  48%|████▊     | 1443/3000 [05:22<04:54,  5.29it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  48%|████▊     | 1448/3000 [05:22<03:19,  7.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  48%|████▊     | 1451/3000 [05:23<04:25,  5.84it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  48%|████▊     | 1452/3000 [05:23<05:51,  4.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  48%|████▊     | 1454/3000 [05:24<05:35,  4.60it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  49%|████▊     | 1456/3000 [05:24<05:01,  5.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  49%|████▊     | 1458/3000 [05:26<09:49,  2.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  49%|████▊     | 1459/3000 [05:26<08:01,  3.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  49%|████▉     | 1463/3000 [05:26<04:53,  5.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  49%|████▉     | 1465/3000 [05:26<03:52,  6.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  49%|████▉     | 1467/3000 [05:27<04:26,  5.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  49%|████▉     | 1468/3000 [05:27<05:27,  4.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  49%|████▉     | 1472/3000 [05:28<05:03,  5.04it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  49%|████▉     | 1473/3000 [05:28<07:29,  3.40it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  49%|████▉     | 1476/3000 [05:29<05:59,  4.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  49%|████▉     | 1477/3000 [05:29<06:24,  3.97it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  49%|████▉     | 1479/3000 [05:30<05:42,  4.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  49%|████▉     | 1483/3000 [05:30<04:21,  5.80it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  49%|████▉     | 1484/3000 [05:31<05:29,  4.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  50%|████▉     | 1487/3000 [05:31<04:33,  5.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  50%|████▉     | 1489/3000 [05:31<04:47,  5.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  50%|████▉     | 1490/3000 [05:32<04:26,  5.66it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  50%|████▉     | 1491/3000 [05:33<11:18,  2.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  50%|████▉     | 1495/3000 [05:33<05:56,  4.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  50%|████▉     | 1496/3000 [05:33<05:16,  4.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  50%|████▉     | 1497/3000 [05:34<05:23,  4.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  50%|████▉     | 1498/3000 [05:34<07:10,  3.49it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  50%|█████     | 1504/3000 [05:35<03:32,  7.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  50%|█████     | 1506/3000 [05:35<04:07,  6.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  50%|█████     | 1508/3000 [05:35<03:43,  6.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  50%|█████     | 1512/3000 [05:36<03:25,  7.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  50%|█████     | 1514/3000 [05:38<09:06,  2.72it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  50%|█████     | 1515/3000 [05:38<09:43,  2.55it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  51%|█████     | 1516/3000 [05:39<09:47,  2.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  51%|█████     | 1517/3000 [05:39<08:47,  2.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  51%|█████     | 1520/3000 [05:39<05:36,  4.40it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  51%|█████     | 1523/3000 [05:40<04:22,  5.63it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  51%|█████     | 1526/3000 [05:40<04:17,  5.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  51%|█████     | 1528/3000 [05:41<04:32,  5.40it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  51%|█████     | 1529/3000 [05:41<04:53,  5.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  51%|█████     | 1531/3000 [05:42<05:51,  4.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  51%|█████     | 1532/3000 [05:42<05:56,  4.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  51%|█████     | 1534/3000 [05:43<05:49,  4.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  51%|█████     | 1535/3000 [05:43<05:02,  4.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  51%|█████     | 1536/3000 [05:43<06:27,  3.78it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  51%|█████▏    | 1538/3000 [05:44<06:14,  3.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  51%|█████▏    | 1539/3000 [05:44<05:44,  4.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  51%|█████▏    | 1543/3000 [05:44<04:40,  5.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  52%|█████▏    | 1545/3000 [05:45<03:29,  6.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  52%|█████▏    | 1546/3000 [05:45<03:49,  6.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  52%|█████▏    | 1548/3000 [05:45<04:20,  5.58it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  52%|█████▏    | 1549/3000 [05:46<04:50,  5.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  52%|█████▏    | 1550/3000 [05:46<04:56,  4.88it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  52%|█████▏    | 1551/3000 [05:46<05:00,  4.83it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  52%|█████▏    | 1554/3000 [05:47<07:00,  3.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  52%|█████▏    | 1556/3000 [05:48<06:18,  3.82it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  52%|█████▏    | 1559/3000 [05:48<04:05,  5.87it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  52%|█████▏    | 1561/3000 [05:48<03:33,  6.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  52%|█████▏    | 1564/3000 [05:48<03:04,  7.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  52%|█████▏    | 1566/3000 [05:49<04:55,  4.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  52%|█████▏    | 1568/3000 [05:49<04:53,  4.87it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  52%|█████▏    | 1570/3000 [05:50<05:03,  4.71it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  52%|█████▏    | 1571/3000 [05:50<05:14,  4.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  52%|█████▏    | 1572/3000 [05:50<05:15,  4.52it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  52%|█████▏    | 1573/3000 [05:51<05:46,  4.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  52%|█████▎    | 1575/3000 [05:51<05:29,  4.32it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  53%|█████▎    | 1576/3000 [05:51<04:42,  5.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  53%|█████▎    | 1577/3000 [05:51<04:48,  4.93it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  53%|█████▎    | 1581/3000 [05:52<02:48,  8.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  53%|█████▎    | 1583/3000 [05:52<03:54,  6.05it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  53%|█████▎    | 1586/3000 [05:53<03:11,  7.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  53%|█████▎    | 1587/3000 [05:53<03:53,  6.04it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  53%|█████▎    | 1588/3000 [05:54<07:02,  3.35it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  53%|█████▎    | 1590/3000 [05:54<05:59,  3.93it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  53%|█████▎    | 1592/3000 [05:54<05:18,  4.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  53%|█████▎    | 1594/3000 [05:54<04:22,  5.36it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  53%|█████▎    | 1598/3000 [05:55<03:25,  6.82it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  53%|█████▎    | 1600/3000 [05:55<03:20,  7.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  53%|█████▎    | 1601/3000 [05:56<04:03,  5.74it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  53%|█████▎    | 1603/3000 [05:56<04:22,  5.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  54%|█████▎    | 1605/3000 [05:56<04:04,  5.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  54%|█████▎    | 1607/3000 [05:57<05:52,  3.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  54%|█████▎    | 1608/3000 [05:57<05:56,  3.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  54%|█████▎    | 1610/3000 [05:58<05:17,  4.37it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  54%|█████▎    | 1611/3000 [05:58<05:19,  4.35it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  54%|█████▍    | 1613/3000 [05:58<04:15,  5.42it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  54%|█████▍    | 1615/3000 [05:59<05:16,  4.38it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  54%|█████▍    | 1618/3000 [05:59<03:38,  6.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  54%|█████▍    | 1620/3000 [05:59<02:52,  7.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  54%|█████▍    | 1622/3000 [06:00<03:17,  6.96it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  54%|█████▍    | 1625/3000 [06:00<04:20,  5.28it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  54%|█████▍    | 1628/3000 [06:01<03:43,  6.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  54%|█████▍    | 1630/3000 [06:02<05:13,  4.37it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  54%|█████▍    | 1631/3000 [06:02<05:28,  4.17it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  54%|█████▍    | 1633/3000 [06:02<04:39,  4.89it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  55%|█████▍    | 1635/3000 [06:02<04:28,  5.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  55%|█████▍    | 1637/3000 [06:03<03:25,  6.63it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  55%|█████▍    | 1640/3000 [06:03<03:00,  7.52it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  55%|█████▍    | 1641/3000 [06:03<03:40,  6.17it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  55%|█████▍    | 1643/3000 [06:04<04:08,  5.47it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  55%|█████▍    | 1645/3000 [06:04<03:31,  6.40it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  55%|█████▍    | 1647/3000 [06:05<05:36,  4.02it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  55%|█████▍    | 1649/3000 [06:05<05:14,  4.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  55%|█████▌    | 1652/3000 [06:06<03:47,  5.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  55%|█████▌    | 1655/3000 [06:06<03:27,  6.48it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  55%|█████▌    | 1656/3000 [06:06<03:17,  6.82it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  55%|█████▌    | 1657/3000 [06:07<04:50,  4.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  55%|█████▌    | 1659/3000 [06:07<03:50,  5.82it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  55%|█████▌    | 1661/3000 [06:07<03:57,  5.63it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  55%|█████▌    | 1663/3000 [06:08<04:15,  5.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  56%|█████▌    | 1665/3000 [06:08<05:54,  3.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  56%|█████▌    | 1668/3000 [06:09<05:22,  4.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  56%|█████▌    | 1672/3000 [06:09<03:05,  7.16it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  56%|█████▌    | 1674/3000 [06:10<02:39,  8.32it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  56%|█████▌    | 1676/3000 [06:10<02:34,  8.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  56%|█████▌    | 1678/3000 [06:10<03:30,  6.27it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  56%|█████▌    | 1679/3000 [06:11<04:00,  5.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  56%|█████▌    | 1680/3000 [06:11<05:21,  4.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  56%|█████▌    | 1682/3000 [06:12<06:04,  3.61it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  56%|█████▌    | 1684/3000 [06:12<04:51,  4.52it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  56%|█████▌    | 1687/3000 [06:13<03:54,  5.61it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  56%|█████▋    | 1689/3000 [06:13<03:03,  7.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  56%|█████▋    | 1691/3000 [06:14<05:02,  4.32it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  56%|█████▋    | 1693/3000 [06:14<03:50,  5.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  56%|█████▋    | 1694/3000 [06:14<04:03,  5.37it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  56%|█████▋    | 1695/3000 [06:14<04:10,  5.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  57%|█████▋    | 1698/3000 [06:15<03:22,  6.42it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  57%|█████▋    | 1700/3000 [06:15<03:48,  5.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  57%|█████▋    | 1702/3000 [06:16<04:24,  4.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  57%|█████▋    | 1705/3000 [06:16<03:07,  6.89it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  57%|█████▋    | 1706/3000 [06:17<06:02,  3.57it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  57%|█████▋    | 1708/3000 [06:17<05:39,  3.80it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  57%|█████▋    | 1710/3000 [06:18<04:29,  4.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  57%|█████▋    | 1712/3000 [06:18<04:11,  5.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  57%|█████▋    | 1715/3000 [06:18<03:00,  7.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  57%|█████▋    | 1716/3000 [06:18<03:36,  5.93it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  57%|█████▋    | 1717/3000 [06:19<04:25,  4.84it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  57%|█████▋    | 1719/3000 [06:19<04:14,  5.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  57%|█████▋    | 1721/3000 [06:19<03:46,  5.66it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  57%|█████▋    | 1722/3000 [06:20<03:43,  5.72it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  57%|█████▋    | 1723/3000 [06:20<04:07,  5.16it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  57%|█████▊    | 1725/3000 [06:21<06:34,  3.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  58%|█████▊    | 1727/3000 [06:21<04:59,  4.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  58%|█████▊    | 1728/3000 [06:21<05:10,  4.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  58%|█████▊    | 1730/3000 [06:22<04:58,  4.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  58%|█████▊    | 1732/3000 [06:22<03:50,  5.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  58%|█████▊    | 1734/3000 [06:23<04:41,  4.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  58%|█████▊    | 1735/3000 [06:23<04:34,  4.60it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  58%|█████▊    | 1736/3000 [06:23<06:02,  3.49it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  58%|█████▊    | 1739/3000 [06:24<04:31,  4.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  58%|█████▊    | 1742/3000 [06:24<04:21,  4.80it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  58%|█████▊    | 1745/3000 [06:25<03:56,  5.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  58%|█████▊    | 1747/3000 [06:25<03:40,  5.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  58%|█████▊    | 1749/3000 [06:26<04:16,  4.88it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  58%|█████▊    | 1752/3000 [06:26<03:47,  5.48it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  58%|█████▊    | 1755/3000 [06:27<04:11,  4.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  59%|█████▊    | 1756/3000 [06:28<05:34,  3.72it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  59%|█████▊    | 1757/3000 [06:28<05:23,  3.84it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  59%|█████▊    | 1760/3000 [06:28<03:35,  5.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  59%|█████▊    | 1761/3000 [06:28<03:19,  6.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  59%|█████▉    | 1763/3000 [06:29<03:38,  5.67it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  59%|█████▉    | 1764/3000 [06:29<04:48,  4.29it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  59%|█████▉    | 1765/3000 [06:29<05:33,  3.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  59%|█████▉    | 1767/3000 [06:30<04:39,  4.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  59%|█████▉    | 1770/3000 [06:30<02:43,  7.52it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  59%|█████▉    | 1771/3000 [06:31<05:43,  3.58it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  59%|█████▉    | 1774/3000 [06:31<05:05,  4.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  59%|█████▉    | 1775/3000 [06:32<04:32,  4.49it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  59%|█████▉    | 1777/3000 [06:32<03:41,  5.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  59%|█████▉    | 1778/3000 [06:32<04:49,  4.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  59%|█████▉    | 1780/3000 [06:33<04:48,  4.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  59%|█████▉    | 1781/3000 [06:34<08:44,  2.32it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  59%|█████▉    | 1783/3000 [06:34<06:13,  3.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  59%|█████▉    | 1784/3000 [06:35<07:07,  2.84it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  60%|█████▉    | 1786/3000 [06:35<06:02,  3.35it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  60%|█████▉    | 1788/3000 [06:35<04:25,  4.57it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  60%|█████▉    | 1790/3000 [06:36<03:50,  5.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  60%|█████▉    | 1792/3000 [06:36<05:31,  3.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  60%|█████▉    | 1795/3000 [06:37<04:48,  4.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  60%|█████▉    | 1797/3000 [06:37<03:30,  5.71it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  60%|██████    | 1803/3000 [06:39<03:49,  5.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  60%|██████    | 1806/3000 [06:39<03:05,  6.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  60%|██████    | 1807/3000 [06:40<05:04,  3.92it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  60%|██████    | 1809/3000 [06:40<05:02,  3.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  60%|██████    | 1810/3000 [06:41<06:23,  3.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  60%|██████    | 1812/3000 [06:41<05:38,  3.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  60%|██████    | 1815/3000 [06:42<04:43,  4.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  61%|██████    | 1816/3000 [06:42<04:12,  4.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  61%|██████    | 1818/3000 [06:42<03:40,  5.36it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  61%|██████    | 1820/3000 [06:43<03:31,  5.58it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  61%|██████    | 1823/3000 [06:43<02:58,  6.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  61%|██████    | 1826/3000 [06:44<03:17,  5.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  61%|██████    | 1828/3000 [06:44<04:23,  4.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  61%|██████    | 1829/3000 [06:44<04:43,  4.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  61%|██████    | 1832/3000 [06:45<04:41,  4.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  61%|██████    | 1835/3000 [06:46<03:57,  4.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  61%|██████    | 1837/3000 [06:46<02:59,  6.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  61%|██████▏   | 1839/3000 [06:46<02:26,  7.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  61%|██████▏   | 1841/3000 [06:47<05:39,  3.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  61%|██████▏   | 1843/3000 [06:48<04:48,  4.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  62%|██████▏   | 1845/3000 [06:48<05:21,  3.60it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  62%|██████▏   | 1847/3000 [06:49<04:18,  4.46it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  62%|██████▏   | 1850/3000 [06:49<03:52,  4.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  62%|██████▏   | 1851/3000 [06:50<04:16,  4.48it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  62%|██████▏   | 1852/3000 [06:50<04:40,  4.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  62%|██████▏   | 1856/3000 [06:50<02:36,  7.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  62%|██████▏   | 1857/3000 [06:50<02:53,  6.60it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  62%|██████▏   | 1858/3000 [06:51<05:06,  3.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  62%|██████▏   | 1859/3000 [06:51<05:33,  3.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  62%|██████▏   | 1862/3000 [06:52<03:41,  5.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  62%|██████▏   | 1866/3000 [06:52<02:36,  7.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  62%|██████▏   | 1868/3000 [06:52<02:11,  8.63it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  62%|██████▏   | 1870/3000 [06:53<03:35,  5.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  62%|██████▏   | 1871/3000 [06:53<03:59,  4.71it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  62%|██████▏   | 1874/3000 [06:54<03:17,  5.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  63%|██████▎   | 1876/3000 [06:54<04:19,  4.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  63%|██████▎   | 1877/3000 [06:55<04:43,  3.97it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  63%|██████▎   | 1880/3000 [06:56<04:56,  3.78it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  63%|██████▎   | 1881/3000 [06:56<04:28,  4.17it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  63%|██████▎   | 1884/3000 [06:56<03:16,  5.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  63%|██████▎   | 1886/3000 [06:57<03:47,  4.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  63%|██████▎   | 1888/3000 [06:57<03:32,  5.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  63%|██████▎   | 1889/3000 [06:58<04:12,  4.40it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  63%|██████▎   | 1892/3000 [06:58<03:10,  5.83it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  63%|██████▎   | 1894/3000 [06:58<02:39,  6.92it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  63%|██████▎   | 1896/3000 [06:59<04:55,  3.74it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  63%|██████▎   | 1897/3000 [06:59<04:29,  4.09it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  63%|██████▎   | 1900/3000 [07:00<03:40,  5.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  63%|██████▎   | 1903/3000 [07:00<02:49,  6.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  63%|██████▎   | 1904/3000 [07:01<05:16,  3.47it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  64%|██████▎   | 1908/3000 [07:01<02:57,  6.16it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  64%|██████▎   | 1909/3000 [07:01<03:02,  5.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  64%|██████▎   | 1912/3000 [07:02<04:33,  3.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  64%|██████▍   | 1913/3000 [07:03<04:21,  4.16it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  64%|██████▍   | 1915/3000 [07:03<04:29,  4.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  64%|██████▍   | 1918/3000 [07:04<03:24,  5.29it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  64%|██████▍   | 1919/3000 [07:04<03:49,  4.71it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  64%|██████▍   | 1922/3000 [07:05<03:28,  5.17it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  64%|██████▍   | 1925/3000 [07:05<03:54,  4.57it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  64%|██████▍   | 1926/3000 [07:06<06:37,  2.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  64%|██████▍   | 1928/3000 [07:07<05:55,  3.02it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  64%|██████▍   | 1931/3000 [07:07<03:40,  4.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  64%|██████▍   | 1934/3000 [07:07<02:43,  6.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  65%|██████▍   | 1936/3000 [07:08<03:56,  4.49it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  65%|██████▍   | 1938/3000 [07:09<03:41,  4.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  65%|██████▍   | 1940/3000 [07:09<03:17,  5.37it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  65%|██████▍   | 1943/3000 [07:09<03:07,  5.63it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  65%|██████▍   | 1944/3000 [07:10<05:52,  3.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  65%|██████▍   | 1945/3000 [07:10<05:23,  3.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  65%|██████▍   | 1946/3000 [07:11<05:03,  3.48it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  65%|██████▍   | 1949/3000 [07:11<03:22,  5.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  65%|██████▌   | 1950/3000 [07:11<04:11,  4.17it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  65%|██████▌   | 1952/3000 [07:12<03:25,  5.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  65%|██████▌   | 1955/3000 [07:12<02:52,  6.05it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  65%|██████▌   | 1956/3000 [07:12<02:50,  6.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  65%|██████▌   | 1957/3000 [07:13<03:04,  5.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  65%|██████▌   | 1958/3000 [07:13<05:26,  3.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  65%|██████▌   | 1959/3000 [07:13<04:59,  3.48it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  65%|██████▌   | 1961/3000 [07:14<04:15,  4.06it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  65%|██████▌   | 1964/3000 [07:14<03:15,  5.29it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  66%|██████▌   | 1968/3000 [07:15<03:18,  5.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  66%|██████▌   | 1969/3000 [07:15<03:00,  5.71it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  66%|██████▌   | 1970/3000 [07:15<03:12,  5.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  66%|██████▌   | 1972/3000 [07:16<03:23,  5.06it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  66%|██████▌   | 1974/3000 [07:16<02:53,  5.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  66%|██████▌   | 1975/3000 [07:16<03:03,  5.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  66%|██████▌   | 1976/3000 [07:17<03:52,  4.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  66%|██████▌   | 1977/3000 [07:17<04:32,  3.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  66%|██████▌   | 1978/3000 [07:18<04:56,  3.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  66%|██████▌   | 1980/3000 [07:18<04:31,  3.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  66%|██████▌   | 1981/3000 [07:18<05:05,  3.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  66%|██████▌   | 1985/3000 [07:19<02:58,  5.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  66%|██████▋   | 1988/3000 [07:19<02:21,  7.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  66%|██████▋   | 1989/3000 [07:20<02:54,  5.80it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  66%|██████▋   | 1993/3000 [07:20<03:03,  5.49it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  66%|██████▋   | 1995/3000 [07:21<03:46,  4.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  67%|██████▋   | 1997/3000 [07:21<03:18,  5.04it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  67%|██████▋   | 2000/3000 [07:22<02:38,  6.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  67%|██████▋   | 2001/3000 [07:22<03:35,  4.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  67%|██████▋   | 2002/3000 [07:23<04:32,  3.66it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  67%|██████▋   | 2004/3000 [07:23<03:46,  4.40it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  67%|██████▋   | 2008/3000 [07:23<02:15,  7.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  67%|██████▋   | 2010/3000 [07:24<02:10,  7.57it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  67%|██████▋   | 2011/3000 [07:24<02:19,  7.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  67%|██████▋   | 2014/3000 [07:25<03:32,  4.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  67%|██████▋   | 2015/3000 [07:25<03:56,  4.17it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  67%|██████▋   | 2020/3000 [07:26<02:12,  7.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  67%|██████▋   | 2022/3000 [07:26<02:13,  7.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  68%|██████▊   | 2026/3000 [07:26<01:54,  8.49it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  68%|██████▊   | 2028/3000 [07:28<04:18,  3.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  68%|██████▊   | 2030/3000 [07:28<04:05,  3.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  68%|██████▊   | 2032/3000 [07:28<03:57,  4.07it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  68%|██████▊   | 2034/3000 [07:29<03:32,  4.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  68%|██████▊   | 2035/3000 [07:29<03:55,  4.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  68%|██████▊   | 2037/3000 [07:29<03:29,  4.60it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  68%|██████▊   | 2040/3000 [07:30<02:35,  6.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  68%|██████▊   | 2043/3000 [07:30<01:52,  8.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  68%|██████▊   | 2046/3000 [07:31<02:35,  6.14it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  68%|██████▊   | 2047/3000 [07:31<02:40,  5.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  68%|██████▊   | 2049/3000 [07:31<02:43,  5.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  68%|██████▊   | 2051/3000 [07:32<03:32,  4.46it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  68%|██████▊   | 2052/3000 [07:32<04:11,  3.77it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  68%|██████▊   | 2054/3000 [07:33<04:02,  3.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  69%|██████▊   | 2056/3000 [07:33<04:14,  3.71it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  69%|██████▊   | 2058/3000 [07:34<02:50,  5.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  69%|██████▊   | 2059/3000 [07:34<02:40,  5.88it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  69%|██████▊   | 2060/3000 [07:34<02:58,  5.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  69%|██████▉   | 2064/3000 [07:34<01:59,  7.86it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  69%|██████▉   | 2065/3000 [07:35<02:04,  7.52it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  69%|██████▉   | 2067/3000 [07:35<01:55,  8.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  69%|██████▉   | 2069/3000 [07:36<03:30,  4.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  69%|██████▉   | 2071/3000 [07:36<02:48,  5.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  69%|██████▉   | 2075/3000 [07:37<02:20,  6.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  69%|██████▉   | 2076/3000 [07:37<03:37,  4.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  69%|██████▉   | 2078/3000 [07:37<03:10,  4.83it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  69%|██████▉   | 2081/3000 [07:38<02:22,  6.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  70%|██████▉   | 2085/3000 [07:38<02:04,  7.35it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  70%|██████▉   | 2087/3000 [07:39<02:48,  5.42it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  70%|██████▉   | 2088/3000 [07:39<02:55,  5.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  70%|██████▉   | 2090/3000 [07:40<03:36,  4.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  70%|██████▉   | 2091/3000 [07:40<03:38,  4.16it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  70%|██████▉   | 2092/3000 [07:40<04:41,  3.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  70%|██████▉   | 2093/3000 [07:41<04:51,  3.11it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  70%|██████▉   | 2095/3000 [07:41<03:45,  4.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  70%|██████▉   | 2096/3000 [07:41<03:31,  4.27it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  70%|██████▉   | 2098/3000 [07:42<03:04,  4.88it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  70%|███████   | 2100/3000 [07:42<03:07,  4.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  70%|███████   | 2102/3000 [07:43<03:12,  4.66it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  70%|███████   | 2105/3000 [07:43<01:51,  8.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  70%|███████   | 2109/3000 [07:44<02:39,  5.57it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  70%|███████   | 2111/3000 [07:44<02:56,  5.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  70%|███████   | 2113/3000 [07:44<02:38,  5.61it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  70%|███████   | 2114/3000 [07:45<04:00,  3.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  70%|███████   | 2115/3000 [07:45<03:55,  3.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  71%|███████   | 2117/3000 [07:46<03:36,  4.07it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  71%|███████   | 2119/3000 [07:46<03:31,  4.16it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  71%|███████   | 2121/3000 [07:46<02:26,  5.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  71%|███████   | 2125/3000 [07:47<01:53,  7.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  71%|███████   | 2128/3000 [07:47<02:04,  6.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  71%|███████   | 2130/3000 [07:48<01:52,  7.74it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  71%|███████   | 2132/3000 [07:49<04:16,  3.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  71%|███████   | 2133/3000 [07:49<03:30,  4.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  71%|███████   | 2135/3000 [07:49<03:53,  3.71it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  71%|███████   | 2137/3000 [07:50<02:29,  5.78it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  71%|███████▏  | 2138/3000 [07:50<02:34,  5.58it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  71%|███████▏  | 2140/3000 [07:50<02:37,  5.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  71%|███████▏  | 2142/3000 [07:50<02:25,  5.92it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  71%|███████▏  | 2144/3000 [07:51<02:19,  6.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  72%|███████▏  | 2146/3000 [07:52<03:23,  4.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  72%|███████▏  | 2150/3000 [07:52<02:21,  5.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  72%|███████▏  | 2151/3000 [07:52<02:23,  5.92it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  72%|███████▏  | 2155/3000 [07:53<02:16,  6.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  72%|███████▏  | 2156/3000 [07:53<02:15,  6.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  72%|███████▏  | 2157/3000 [07:53<02:31,  5.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  72%|███████▏  | 2159/3000 [07:53<02:14,  6.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  72%|███████▏  | 2162/3000 [07:54<01:50,  7.60it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  72%|███████▏  | 2163/3000 [07:54<02:06,  6.60it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  72%|███████▏  | 2166/3000 [07:55<03:00,  4.61it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  72%|███████▏  | 2169/3000 [07:56<02:46,  4.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  72%|███████▏  | 2170/3000 [07:56<03:05,  4.47it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  72%|███████▏  | 2171/3000 [07:56<03:33,  3.88it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  72%|███████▏  | 2172/3000 [07:56<03:31,  3.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  73%|███████▎  | 2177/3000 [07:57<02:00,  6.82it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  73%|███████▎  | 2179/3000 [07:57<02:03,  6.66it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  73%|███████▎  | 2182/3000 [07:58<02:18,  5.89it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  73%|███████▎  | 2184/3000 [07:58<02:11,  6.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  73%|███████▎  | 2186/3000 [07:59<02:03,  6.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  73%|███████▎  | 2187/3000 [07:59<02:54,  4.66it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  73%|███████▎  | 2188/3000 [07:59<03:26,  3.93it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  73%|███████▎  | 2189/3000 [08:00<04:37,  2.92it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  73%|███████▎  | 2191/3000 [08:01<03:50,  3.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  73%|███████▎  | 2193/3000 [08:01<02:58,  4.52it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  73%|███████▎  | 2194/3000 [08:01<02:41,  4.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  73%|███████▎  | 2196/3000 [08:01<02:20,  5.74it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  73%|███████▎  | 2199/3000 [08:02<01:44,  7.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  73%|███████▎  | 2202/3000 [08:02<01:38,  8.09it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  74%|███████▎  | 2205/3000 [08:02<01:15, 10.49it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  74%|███████▎  | 2207/3000 [08:03<02:27,  5.37it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  74%|███████▎  | 2209/3000 [08:04<02:54,  4.52it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  74%|███████▎  | 2210/3000 [08:04<02:52,  4.57it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  74%|███████▍  | 2213/3000 [08:05<02:53,  4.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  74%|███████▍  | 2214/3000 [08:05<02:46,  4.71it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  74%|███████▍  | 2217/3000 [08:05<02:13,  5.86it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  74%|███████▍  | 2221/3000 [08:06<01:42,  7.63it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  74%|███████▍  | 2223/3000 [08:06<02:18,  5.61it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  74%|███████▍  | 2224/3000 [08:07<02:54,  4.46it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  74%|███████▍  | 2227/3000 [08:07<02:09,  5.96it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  74%|███████▍  | 2229/3000 [08:07<02:18,  5.58it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  74%|███████▍  | 2231/3000 [08:08<01:57,  6.55it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  74%|███████▍  | 2233/3000 [08:08<02:04,  6.16it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  74%|███████▍  | 2234/3000 [08:08<03:02,  4.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  75%|███████▍  | 2238/3000 [08:09<01:49,  6.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  75%|███████▍  | 2240/3000 [08:09<02:35,  4.89it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  75%|███████▍  | 2242/3000 [08:10<02:29,  5.07it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  75%|███████▍  | 2243/3000 [08:10<02:21,  5.36it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  75%|███████▍  | 2244/3000 [08:10<03:07,  4.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  75%|███████▍  | 2248/3000 [08:11<02:01,  6.17it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  75%|███████▌  | 2250/3000 [08:11<01:34,  7.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  75%|███████▌  | 2252/3000 [08:12<02:27,  5.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  75%|███████▌  | 2254/3000 [08:12<02:00,  6.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  75%|███████▌  | 2255/3000 [08:12<02:30,  4.96it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  75%|███████▌  | 2256/3000 [08:12<02:34,  4.82it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  75%|███████▌  | 2257/3000 [08:13<03:41,  3.36it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  75%|███████▌  | 2259/3000 [08:13<03:08,  3.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  75%|███████▌  | 2262/3000 [08:14<01:43,  7.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  75%|███████▌  | 2264/3000 [08:14<02:27,  4.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  76%|███████▌  | 2267/3000 [08:15<01:58,  6.17it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  76%|███████▌  | 2270/3000 [08:15<02:10,  5.60it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  76%|███████▌  | 2272/3000 [08:15<01:39,  7.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  76%|███████▌  | 2275/3000 [08:16<01:49,  6.60it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  76%|███████▌  | 2277/3000 [08:17<02:49,  4.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  76%|███████▌  | 2280/3000 [08:17<02:47,  4.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  76%|███████▌  | 2281/3000 [08:18<03:03,  3.93it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  76%|███████▌  | 2283/3000 [08:18<02:45,  4.32it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  76%|███████▌  | 2286/3000 [08:19<02:21,  5.04it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  76%|███████▋  | 2290/3000 [08:19<01:47,  6.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  76%|███████▋  | 2292/3000 [08:20<02:06,  5.61it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  76%|███████▋  | 2293/3000 [08:20<02:48,  4.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  77%|███████▋  | 2296/3000 [08:21<02:31,  4.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  77%|███████▋  | 2297/3000 [08:21<02:35,  4.52it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  77%|███████▋  | 2300/3000 [08:22<02:04,  5.60it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  77%|███████▋  | 2302/3000 [08:22<02:19,  4.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  77%|███████▋  | 2303/3000 [08:22<02:49,  4.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  77%|███████▋  | 2304/3000 [08:23<03:24,  3.40it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  77%|███████▋  | 2305/3000 [08:23<03:33,  3.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  77%|███████▋  | 2307/3000 [08:24<02:55,  3.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  77%|███████▋  | 2309/3000 [08:24<02:44,  4.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  77%|███████▋  | 2312/3000 [08:24<02:16,  5.06it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  77%|███████▋  | 2314/3000 [08:25<02:13,  5.14it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  77%|███████▋  | 2315/3000 [08:25<03:04,  3.71it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  77%|███████▋  | 2319/3000 [08:26<01:53,  6.02it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  77%|███████▋  | 2321/3000 [08:26<01:43,  6.55it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  77%|███████▋  | 2323/3000 [08:27<02:07,  5.32it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  78%|███████▊  | 2325/3000 [08:27<01:40,  6.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  78%|███████▊  | 2328/3000 [08:27<01:44,  6.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  78%|███████▊  | 2331/3000 [08:28<01:38,  6.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  78%|███████▊  | 2333/3000 [08:28<02:18,  4.80it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  78%|███████▊  | 2334/3000 [08:29<02:22,  4.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  78%|███████▊  | 2337/3000 [08:29<02:45,  4.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  78%|███████▊  | 2340/3000 [08:30<01:52,  5.87it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  78%|███████▊  | 2343/3000 [08:30<01:38,  6.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  78%|███████▊  | 2345/3000 [08:30<01:34,  6.92it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  78%|███████▊  | 2349/3000 [08:31<01:20,  8.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  78%|███████▊  | 2352/3000 [08:31<01:34,  6.86it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  78%|███████▊  | 2353/3000 [08:32<03:08,  3.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  78%|███████▊  | 2355/3000 [08:33<02:48,  3.83it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  79%|███████▊  | 2357/3000 [08:33<02:22,  4.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  79%|███████▊  | 2359/3000 [08:34<02:23,  4.48it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  79%|███████▊  | 2360/3000 [08:34<02:45,  3.87it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  79%|███████▊  | 2362/3000 [08:34<02:30,  4.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  79%|███████▉  | 2366/3000 [08:35<01:33,  6.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  79%|███████▉  | 2367/3000 [08:35<01:38,  6.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  79%|███████▉  | 2368/3000 [08:35<01:46,  5.96it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  79%|███████▉  | 2370/3000 [08:35<01:32,  6.84it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  79%|███████▉  | 2371/3000 [08:36<01:45,  5.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  79%|███████▉  | 2374/3000 [08:36<01:55,  5.42it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  79%|███████▉  | 2376/3000 [08:37<02:12,  4.72it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  79%|███████▉  | 2377/3000 [08:37<02:44,  3.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  79%|███████▉  | 2380/3000 [08:38<02:03,  5.04it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  79%|███████▉  | 2381/3000 [08:38<02:48,  3.67it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  79%|███████▉  | 2382/3000 [08:39<02:43,  3.78it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  80%|███████▉  | 2388/3000 [08:39<01:22,  7.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  80%|███████▉  | 2391/3000 [08:39<01:17,  7.86it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  80%|███████▉  | 2392/3000 [08:40<01:27,  6.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  80%|███████▉  | 2393/3000 [08:40<01:49,  5.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  80%|███████▉  | 2395/3000 [08:40<01:33,  6.46it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  80%|███████▉  | 2397/3000 [08:41<01:41,  5.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  80%|███████▉  | 2398/3000 [08:41<01:59,  5.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  80%|███████▉  | 2399/3000 [08:41<02:16,  4.42it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  80%|████████  | 2400/3000 [08:42<03:25,  2.92it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  80%|████████  | 2403/3000 [08:42<02:09,  4.61it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  80%|████████  | 2406/3000 [08:43<01:46,  5.57it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  80%|████████  | 2408/3000 [08:43<01:38,  5.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  80%|████████  | 2412/3000 [08:43<01:17,  7.60it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  80%|████████  | 2413/3000 [08:44<01:24,  6.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  80%|████████  | 2414/3000 [08:44<02:23,  4.09it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  81%|████████  | 2416/3000 [08:45<02:06,  4.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  81%|████████  | 2417/3000 [08:45<02:11,  4.42it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  81%|████████  | 2419/3000 [08:45<02:02,  4.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  81%|████████  | 2420/3000 [08:46<02:56,  3.29it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  81%|████████  | 2423/3000 [08:46<01:54,  5.02it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  81%|████████  | 2424/3000 [08:46<01:59,  4.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  81%|████████  | 2427/3000 [08:47<01:38,  5.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  81%|████████  | 2429/3000 [08:47<01:41,  5.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  81%|████████  | 2433/3000 [08:47<01:02,  9.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  81%|████████  | 2435/3000 [08:49<02:38,  3.57it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  81%|████████  | 2437/3000 [08:49<02:47,  3.36it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  81%|████████▏ | 2440/3000 [08:50<02:35,  3.61it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  81%|████████▏ | 2444/3000 [08:50<01:22,  6.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  82%|████████▏ | 2446/3000 [08:51<01:13,  7.57it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  82%|████████▏ | 2448/3000 [08:51<01:11,  7.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  82%|████████▏ | 2450/3000 [08:51<01:19,  6.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  82%|████████▏ | 2452/3000 [08:51<01:08,  8.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  82%|████████▏ | 2454/3000 [08:52<01:41,  5.35it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  82%|████████▏ | 2455/3000 [08:53<03:21,  2.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  82%|████████▏ | 2457/3000 [08:54<03:08,  2.87it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  82%|████████▏ | 2459/3000 [08:54<02:51,  3.16it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  82%|████████▏ | 2462/3000 [08:55<02:23,  3.74it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  82%|████████▏ | 2466/3000 [08:55<01:43,  5.16it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  82%|████████▏ | 2469/3000 [08:56<01:18,  6.72it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  82%|████████▏ | 2473/3000 [08:56<01:15,  7.02it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  82%|████████▎ | 2475/3000 [08:57<01:18,  6.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  83%|████████▎ | 2476/3000 [08:57<01:25,  6.16it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  83%|████████▎ | 2477/3000 [08:58<02:33,  3.40it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  83%|████████▎ | 2479/3000 [08:58<01:58,  4.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  83%|████████▎ | 2480/3000 [08:58<02:14,  3.87it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  83%|████████▎ | 2481/3000 [08:59<02:09,  4.02it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  83%|████████▎ | 2483/3000 [08:59<02:14,  3.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  83%|████████▎ | 2486/3000 [09:00<01:41,  5.06it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  83%|████████▎ | 2488/3000 [09:00<01:44,  4.89it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  83%|████████▎ | 2490/3000 [09:00<01:51,  4.58it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  83%|████████▎ | 2491/3000 [09:01<01:37,  5.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  83%|████████▎ | 2492/3000 [09:01<01:55,  4.40it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  83%|████████▎ | 2497/3000 [09:01<01:13,  6.86it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  83%|████████▎ | 2498/3000 [09:02<01:15,  6.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  83%|████████▎ | 2499/3000 [09:02<01:23,  6.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  83%|████████▎ | 2502/3000 [09:03<01:45,  4.72it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  84%|████████▎ | 2505/3000 [09:03<01:38,  5.02it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  84%|████████▎ | 2507/3000 [09:04<01:25,  5.77it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  84%|████████▎ | 2508/3000 [09:04<01:20,  6.11it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  84%|████████▎ | 2509/3000 [09:04<02:25,  3.38it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  84%|████████▎ | 2511/3000 [09:05<02:01,  4.02it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  84%|████████▍ | 2513/3000 [09:05<01:46,  4.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  84%|████████▍ | 2517/3000 [09:05<01:04,  7.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  84%|████████▍ | 2519/3000 [09:06<01:02,  7.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  84%|████████▍ | 2521/3000 [09:06<01:07,  7.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  84%|████████▍ | 2523/3000 [09:07<01:26,  5.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  84%|████████▍ | 2524/3000 [09:07<01:37,  4.87it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  84%|████████▍ | 2525/3000 [09:07<01:38,  4.82it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  84%|████████▍ | 2527/3000 [09:07<01:33,  5.07it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  84%|████████▍ | 2529/3000 [09:08<01:31,  5.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  84%|████████▍ | 2531/3000 [09:08<01:26,  5.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  84%|████████▍ | 2532/3000 [09:09<02:12,  3.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  84%|████████▍ | 2534/3000 [09:09<02:16,  3.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  85%|████████▍ | 2538/3000 [09:10<01:17,  5.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  85%|████████▍ | 2540/3000 [09:10<00:58,  7.84it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  85%|████████▍ | 2544/3000 [09:10<00:48,  9.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  85%|████████▍ | 2546/3000 [09:11<00:55,  8.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  85%|████████▍ | 2548/3000 [09:12<01:31,  4.96it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  85%|████████▌ | 2550/3000 [09:12<01:29,  5.02it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  85%|████████▌ | 2551/3000 [09:12<01:26,  5.17it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  85%|████████▌ | 2555/3000 [09:13<01:16,  5.80it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  85%|████████▌ | 2557/3000 [09:13<01:09,  6.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  85%|████████▌ | 2559/3000 [09:14<01:39,  4.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  85%|████████▌ | 2561/3000 [09:14<01:09,  6.36it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  86%|████████▌ | 2565/3000 [09:14<01:03,  6.84it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  86%|████████▌ | 2569/3000 [09:15<01:00,  7.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  86%|████████▌ | 2571/3000 [09:16<01:43,  4.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  86%|████████▌ | 2573/3000 [09:16<01:30,  4.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  86%|████████▌ | 2576/3000 [09:17<01:13,  5.77it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  86%|████████▌ | 2577/3000 [09:17<01:49,  3.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  86%|████████▌ | 2579/3000 [09:18<01:47,  3.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  86%|████████▌ | 2582/3000 [09:18<01:27,  4.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  86%|████████▌ | 2583/3000 [09:19<01:30,  4.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  86%|████████▌ | 2584/3000 [09:19<02:15,  3.06it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  86%|████████▋ | 2588/3000 [09:20<01:28,  4.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  86%|████████▋ | 2589/3000 [09:20<01:43,  3.97it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  86%|████████▋ | 2592/3000 [09:21<01:11,  5.67it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  86%|████████▋ | 2594/3000 [09:21<01:18,  5.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  86%|████████▋ | 2595/3000 [09:21<01:30,  4.47it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  87%|████████▋ | 2596/3000 [09:22<01:36,  4.17it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  87%|████████▋ | 2598/3000 [09:22<01:28,  4.55it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  87%|████████▋ | 2601/3000 [09:23<01:18,  5.11it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  87%|████████▋ | 2603/3000 [09:23<01:42,  3.89it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  87%|████████▋ | 2605/3000 [09:24<01:43,  3.82it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  87%|████████▋ | 2606/3000 [09:24<01:49,  3.58it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  87%|████████▋ | 2608/3000 [09:25<01:26,  4.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  87%|████████▋ | 2609/3000 [09:25<01:26,  4.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  87%|████████▋ | 2614/3000 [09:25<00:54,  7.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  87%|████████▋ | 2617/3000 [09:26<01:05,  5.89it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  87%|████████▋ | 2618/3000 [09:26<01:12,  5.28it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  87%|████████▋ | 2621/3000 [09:27<01:28,  4.28it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  87%|████████▋ | 2623/3000 [09:27<01:21,  4.60it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  88%|████████▊ | 2625/3000 [09:28<01:04,  5.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  88%|████████▊ | 2626/3000 [09:28<01:09,  5.36it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  88%|████████▊ | 2627/3000 [09:28<01:14,  4.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  88%|████████▊ | 2631/3000 [09:29<00:51,  7.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  88%|████████▊ | 2634/3000 [09:29<01:04,  5.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  88%|████████▊ | 2635/3000 [09:30<01:06,  5.48it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  88%|████████▊ | 2636/3000 [09:30<01:19,  4.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  88%|████████▊ | 2637/3000 [09:30<01:28,  4.09it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  88%|████████▊ | 2638/3000 [09:31<01:37,  3.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  88%|████████▊ | 2641/3000 [09:31<01:16,  4.67it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  88%|████████▊ | 2642/3000 [09:31<01:08,  5.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  88%|████████▊ | 2644/3000 [09:32<01:08,  5.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  88%|████████▊ | 2647/3000 [09:32<00:55,  6.38it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  88%|████████▊ | 2648/3000 [09:32<01:00,  5.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  88%|████████▊ | 2650/3000 [09:32<00:55,  6.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  88%|████████▊ | 2651/3000 [09:33<01:23,  4.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  88%|████████▊ | 2654/3000 [09:33<01:02,  5.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  88%|████████▊ | 2655/3000 [09:34<01:31,  3.77it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  89%|████████▊ | 2656/3000 [09:34<01:29,  3.84it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  89%|████████▊ | 2657/3000 [09:35<01:41,  3.37it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  89%|████████▊ | 2661/3000 [09:35<00:53,  6.29it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  89%|████████▊ | 2662/3000 [09:35<01:21,  4.16it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  89%|████████▉ | 2664/3000 [09:36<01:11,  4.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  89%|████████▉ | 2668/3000 [09:36<00:50,  6.61it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  89%|████████▉ | 2670/3000 [09:37<00:51,  6.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  89%|████████▉ | 2673/3000 [09:37<01:00,  5.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  89%|████████▉ | 2676/3000 [09:38<00:53,  6.09it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  89%|████████▉ | 2677/3000 [09:38<00:51,  6.29it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  89%|████████▉ | 2679/3000 [09:38<01:05,  4.88it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  89%|████████▉ | 2680/3000 [09:39<01:15,  4.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  89%|████████▉ | 2681/3000 [09:39<01:20,  3.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  89%|████████▉ | 2683/3000 [09:39<01:00,  5.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  89%|████████▉ | 2684/3000 [09:40<01:15,  4.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  90%|████████▉ | 2686/3000 [09:40<00:58,  5.37it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  90%|████████▉ | 2689/3000 [09:40<00:48,  6.48it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  90%|████████▉ | 2690/3000 [09:41<01:06,  4.63it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  90%|████████▉ | 2693/3000 [09:41<01:00,  5.09it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  90%|████████▉ | 2696/3000 [09:42<00:41,  7.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  90%|████████▉ | 2698/3000 [09:42<00:38,  7.84it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  90%|████████▉ | 2699/3000 [09:42<00:54,  5.55it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  90%|█████████ | 2703/3000 [09:43<00:51,  5.74it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  90%|█████████ | 2704/3000 [09:43<00:55,  5.38it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  90%|█████████ | 2705/3000 [09:44<01:09,  4.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  90%|█████████ | 2708/3000 [09:44<00:51,  5.67it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  90%|█████████ | 2709/3000 [09:44<00:50,  5.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  90%|█████████ | 2713/3000 [09:45<00:55,  5.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  90%|█████████ | 2714/3000 [09:45<00:50,  5.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  90%|█████████ | 2715/3000 [09:46<01:03,  4.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  91%|█████████ | 2716/3000 [09:46<01:23,  3.42it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  91%|█████████ | 2719/3000 [09:47<00:56,  4.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  91%|█████████ | 2721/3000 [09:47<00:59,  4.67it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  91%|█████████ | 2724/3000 [09:47<00:42,  6.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  91%|█████████ | 2725/3000 [09:48<01:16,  3.57it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  91%|█████████ | 2730/3000 [09:49<00:43,  6.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  91%|█████████ | 2731/3000 [09:49<00:55,  4.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  91%|█████████ | 2736/3000 [09:50<00:42,  6.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  91%|█████████▏| 2738/3000 [09:50<00:53,  4.87it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  91%|█████████▏| 2741/3000 [09:51<00:57,  4.49it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  91%|█████████▏| 2743/3000 [09:51<00:46,  5.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  92%|█████████▏| 2745/3000 [09:52<00:52,  4.87it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  92%|█████████▏| 2747/3000 [09:53<01:04,  3.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  92%|█████████▏| 2748/3000 [09:53<01:05,  3.87it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  92%|█████████▏| 2752/3000 [09:53<00:40,  6.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  92%|█████████▏| 2754/3000 [09:54<00:37,  6.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  92%|█████████▏| 2755/3000 [09:54<00:42,  5.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  92%|█████████▏| 2756/3000 [09:54<01:07,  3.63it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  92%|█████████▏| 2759/3000 [09:55<00:55,  4.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  92%|█████████▏| 2760/3000 [09:55<00:57,  4.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  92%|█████████▏| 2762/3000 [09:56<00:49,  4.80it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  92%|█████████▏| 2765/3000 [09:56<00:44,  5.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  92%|█████████▏| 2768/3000 [09:56<00:28,  8.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  92%|█████████▏| 2770/3000 [09:57<00:38,  5.97it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  92%|█████████▏| 2772/3000 [09:57<00:34,  6.66it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  92%|█████████▏| 2773/3000 [09:57<00:38,  5.88it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  92%|█████████▎| 2775/3000 [09:58<00:40,  5.57it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  93%|█████████▎| 2776/3000 [09:58<00:47,  4.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  93%|█████████▎| 2777/3000 [09:59<01:08,  3.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  93%|█████████▎| 2779/3000 [09:59<01:13,  2.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  93%|█████████▎| 2780/3000 [10:00<01:11,  3.09it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  93%|█████████▎| 2785/3000 [10:00<00:36,  5.89it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  93%|█████████▎| 2786/3000 [10:00<00:34,  6.27it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  93%|█████████▎| 2790/3000 [10:01<00:24,  8.40it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  93%|█████████▎| 2792/3000 [10:01<00:37,  5.48it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  93%|█████████▎| 2793/3000 [10:01<00:39,  5.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  93%|█████████▎| 2794/3000 [10:02<00:41,  5.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  93%|█████████▎| 2796/3000 [10:02<00:43,  4.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  93%|█████████▎| 2797/3000 [10:02<00:37,  5.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  93%|█████████▎| 2800/3000 [10:03<00:35,  5.63it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  93%|█████████▎| 2801/3000 [10:03<00:33,  5.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  93%|█████████▎| 2803/3000 [10:03<00:41,  4.78it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  94%|█████████▎| 2805/3000 [10:04<00:46,  4.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  94%|█████████▎| 2807/3000 [10:04<00:42,  4.55it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  94%|█████████▎| 2809/3000 [10:05<00:36,  5.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  94%|█████████▎| 2811/3000 [10:05<00:39,  4.78it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  94%|█████████▍| 2814/3000 [10:06<00:31,  5.96it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  94%|█████████▍| 2816/3000 [10:06<00:25,  7.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  94%|█████████▍| 2817/3000 [10:07<00:51,  3.55it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  94%|█████████▍| 2818/3000 [10:07<00:51,  3.57it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  94%|█████████▍| 2820/3000 [10:07<00:39,  4.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  94%|█████████▍| 2821/3000 [10:07<00:42,  4.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  94%|█████████▍| 2825/3000 [10:08<00:28,  6.16it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  94%|█████████▍| 2827/3000 [10:09<00:37,  4.66it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  94%|█████████▍| 2829/3000 [10:09<00:36,  4.66it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  94%|█████████▍| 2830/3000 [10:10<00:47,  3.57it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  94%|█████████▍| 2833/3000 [10:10<00:38,  4.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  94%|█████████▍| 2835/3000 [10:10<00:27,  5.93it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  95%|█████████▍| 2836/3000 [10:11<00:29,  5.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  95%|█████████▍| 2838/3000 [10:11<00:34,  4.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  95%|█████████▍| 2840/3000 [10:12<00:36,  4.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  95%|█████████▍| 2841/3000 [10:12<00:36,  4.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  95%|█████████▍| 2844/3000 [10:12<00:27,  5.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  95%|█████████▍| 2846/3000 [10:13<00:27,  5.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  95%|█████████▍| 2847/3000 [10:13<00:35,  4.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  95%|█████████▍| 2848/3000 [10:14<00:47,  3.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  95%|█████████▌| 2850/3000 [10:14<00:39,  3.84it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  95%|█████████▌| 2851/3000 [10:14<00:41,  3.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  95%|█████████▌| 2853/3000 [10:15<00:32,  4.55it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  95%|█████████▌| 2855/3000 [10:15<00:38,  3.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  95%|█████████▌| 2857/3000 [10:16<00:37,  3.80it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  95%|█████████▌| 2859/3000 [10:16<00:33,  4.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  95%|█████████▌| 2861/3000 [10:17<00:31,  4.47it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  96%|█████████▌| 2865/3000 [10:17<00:22,  6.05it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  96%|█████████▌| 2867/3000 [10:18<00:26,  5.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  96%|█████████▌| 2868/3000 [10:18<00:26,  4.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  96%|█████████▌| 2871/3000 [10:18<00:25,  5.04it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  96%|█████████▌| 2872/3000 [10:19<00:25,  4.96it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  96%|█████████▌| 2874/3000 [10:19<00:30,  4.09it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  96%|█████████▌| 2876/3000 [10:19<00:22,  5.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  96%|█████████▌| 2880/3000 [10:20<00:15,  7.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  96%|█████████▌| 2881/3000 [10:20<00:17,  6.88it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  96%|█████████▌| 2883/3000 [10:20<00:16,  7.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  96%|█████████▌| 2884/3000 [10:21<00:18,  6.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  96%|█████████▌| 2887/3000 [10:21<00:16,  6.66it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  96%|█████████▋| 2888/3000 [10:21<00:18,  6.06it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  96%|█████████▋| 2889/3000 [10:22<00:39,  2.84it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  96%|█████████▋| 2891/3000 [10:23<00:37,  2.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  96%|█████████▋| 2893/3000 [10:23<00:27,  3.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  97%|█████████▋| 2897/3000 [10:23<00:16,  6.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  97%|█████████▋| 2900/3000 [10:24<00:11,  8.57it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  97%|█████████▋| 2903/3000 [10:25<00:20,  4.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  97%|█████████▋| 2905/3000 [10:25<00:16,  5.89it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  97%|█████████▋| 2906/3000 [10:26<00:29,  3.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  97%|█████████▋| 2908/3000 [10:26<00:22,  4.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  97%|█████████▋| 2909/3000 [10:27<00:26,  3.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  97%|█████████▋| 2913/3000 [10:27<00:14,  6.02it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  97%|█████████▋| 2916/3000 [10:28<00:14,  5.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  97%|█████████▋| 2920/3000 [10:28<00:11,  6.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  97%|█████████▋| 2924/3000 [10:29<00:16,  4.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  98%|█████████▊| 2928/3000 [10:30<00:14,  4.87it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  98%|█████████▊| 2930/3000 [10:31<00:13,  5.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  98%|█████████▊| 2933/3000 [10:31<00:11,  5.67it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  98%|█████████▊| 2935/3000 [10:31<00:10,  6.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  98%|█████████▊| 2936/3000 [10:32<00:13,  4.61it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  98%|█████████▊| 2939/3000 [10:32<00:09,  6.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  98%|█████████▊| 2941/3000 [10:33<00:18,  3.16it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  98%|█████████▊| 2943/3000 [10:34<00:13,  4.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  98%|█████████▊| 2944/3000 [10:34<00:13,  4.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  98%|█████████▊| 2945/3000 [10:34<00:14,  3.87it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  98%|█████████▊| 2946/3000 [10:34<00:13,  4.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  98%|█████████▊| 2949/3000 [10:35<00:09,  5.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  98%|█████████▊| 2950/3000 [10:35<00:09,  5.05it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  98%|█████████▊| 2953/3000 [10:35<00:06,  6.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  98%|█████████▊| 2955/3000 [10:36<00:06,  7.42it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  99%|█████████▊| 2956/3000 [10:36<00:08,  5.49it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  99%|█████████▊| 2958/3000 [10:36<00:06,  6.29it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  99%|█████████▊| 2959/3000 [10:36<00:07,  5.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  99%|█████████▊| 2960/3000 [10:37<00:10,  3.78it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  99%|█████████▊| 2961/3000 [10:38<00:14,  2.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  99%|█████████▊| 2962/3000 [10:38<00:14,  2.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  99%|█████████▉| 2965/3000 [10:38<00:07,  4.46it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  99%|█████████▉| 2966/3000 [10:39<00:08,  3.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  99%|█████████▉| 2968/3000 [10:39<00:06,  4.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  99%|█████████▉| 2970/3000 [10:39<00:05,  5.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  99%|█████████▉| 2972/3000 [10:40<00:05,  5.40it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  99%|█████████▉| 2973/3000 [10:40<00:04,  5.97it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  99%|█████████▉| 2975/3000 [10:40<00:03,  7.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  99%|█████████▉| 2977/3000 [10:40<00:03,  7.48it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  99%|█████████▉| 2978/3000 [10:41<00:04,  4.55it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  99%|█████████▉| 2980/3000 [10:41<00:04,  4.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  99%|█████████▉| 2982/3000 [10:42<00:05,  3.14it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  99%|█████████▉| 2983/3000 [10:43<00:06,  2.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow: 100%|█████████▉| 2986/3000 [10:43<00:03,  4.49it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow: 100%|█████████▉| 2989/3000 [10:44<00:02,  5.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow: 100%|█████████▉| 2991/3000 [10:44<00:01,  5.84it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow: 100%|█████████▉| 2994/3000 [10:44<00:01,  4.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow: 100%|█████████▉| 2996/3000 [10:45<00:00,  4.74it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow: 100%|██████████| 3000/3000 [10:45<00:00,  6.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow: 100%|██████████| 3000/3000 [10:45<00:00,  4.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2026-01-04 09:12:31.700\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36m__main__\u001b[0m:\u001b[36m<module>\u001b[0m:\u001b[36m4\u001b[0m - \u001b[1mEvaluation metrics (before optimization): {'f1': 0.0, 'em': 0.0, 'acc': 0.42133333333333334}\u001b[0m\n",
      "\u001b[32m2026-01-04 09:12:31.700\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36m__main__\u001b[0m:\u001b[36m<module>\u001b[0m:\u001b[36m6\u001b[0m - \u001b[1mOptimizing workflow...\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "  0%|          | 0/20 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2026-01-04 09:12:31.705\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36mstep\u001b[0m:\u001b[36m319\u001b[0m - \u001b[1mExecuting workflow...\u001b[0m\n",
      "\u001b[32m2026-01-04 09:12:55.082\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36mstep\u001b[0m:\u001b[36m347\u001b[0m - \u001b[1mComputing gradients...\u001b[0m\n",
      "\u001b[32m2026-01-04 09:14:27.340\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36mstep\u001b[0m:\u001b[36m349\u001b[0m - \u001b[1mUpdating agents...\u001b[0m\n",
      "\u001b[32m2026-01-04 09:14:33.762\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36mstep\u001b[0m:\u001b[36m353\u001b[0m - \u001b[1mAgents updated\u001b[0m\n",
      "\u001b[32m2026-01-04 09:14:33.762\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m271\u001b[0m - \u001b[1mEvaluating the workflow at step 1 ...\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:   0%|          | 0/50 [00:00<?, ?it/s]\u001b[A\n",
      "Evaluating workflow:   2%|▏         | 1/50 [00:01<01:20,  1.65s/it]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:   4%|▍         | 2/50 [00:02<00:54,  1.13s/it]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:   6%|▌         | 3/50 [00:02<00:35,  1.34it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:   8%|▊         | 4/50 [00:03<00:27,  1.65it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  14%|█▍        | 7/50 [00:03<00:12,  3.34it/s]\u001b[A\n",
      "Evaluating workflow:  16%|█▌        | 8/50 [00:03<00:10,  3.82it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  22%|██▏       | 11/50 [00:03<00:06,  6.50it/s]\u001b[A\n",
      "Evaluating workflow:  34%|███▍      | 17/50 [00:03<00:02, 13.46it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  40%|████      | 20/50 [00:04<00:03,  8.98it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  44%|████▍     | 22/50 [00:05<00:05,  4.84it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  48%|████▊     | 24/50 [00:05<00:05,  5.06it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  52%|█████▏    | 26/50 [00:06<00:05,  4.18it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  54%|█████▍    | 27/50 [00:06<00:05,  4.17it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  58%|█████▊    | 29/50 [00:07<00:04,  4.72it/s]\u001b[A\n",
      "Evaluating workflow:  64%|██████▍   | 32/50 [00:07<00:02,  7.03it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  68%|██████▊   | 34/50 [00:07<00:01,  8.32it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  72%|███████▏  | 36/50 [00:07<00:01,  8.31it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  78%|███████▊  | 39/50 [00:07<00:01,  9.23it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  82%|████████▏ | 41/50 [00:08<00:01,  8.83it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  86%|████████▌ | 43/50 [00:08<00:01,  5.18it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  88%|████████▊ | 44/50 [00:09<00:01,  4.67it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  90%|█████████ | 45/50 [00:09<00:01,  3.57it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  94%|█████████▍| 47/50 [00:10<00:00,  4.12it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  98%|█████████▊| 49/50 [00:10<00:00,  4.51it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow: 100%|██████████| 50/50 [00:12<00:00,  4.12it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "\u001b[32m2026-01-04 09:14:46.042\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m275\u001b[0m - \u001b[1mStep 1 metrics: {'f1': 0.0, 'em': 0.0, 'acc': 0.52}\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "\r",
      "  5%|▌         | 1/20 [02:14<42:32, 134.34s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2026-01-04 09:14:46.043\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36mstep\u001b[0m:\u001b[36m319\u001b[0m - \u001b[1mExecuting workflow...\u001b[0m\n",
      "\u001b[32m2026-01-04 09:15:01.483\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36mstep\u001b[0m:\u001b[36m347\u001b[0m - \u001b[1mComputing gradients...\u001b[0m\n",
      "\u001b[32m2026-01-04 09:16:30.832\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36mstep\u001b[0m:\u001b[36m349\u001b[0m - \u001b[1mUpdating agents...\u001b[0m\n",
      "\u001b[32m2026-01-04 09:16:36.933\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36mstep\u001b[0m:\u001b[36m353\u001b[0m - \u001b[1mAgents updated\u001b[0m\n",
      "\u001b[32m2026-01-04 09:16:36.933\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m271\u001b[0m - \u001b[1mEvaluating the workflow at step 2 ...\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:   0%|          | 0/50 [00:00<?, ?it/s]\u001b[A\n",
      "Evaluating workflow:   2%|▏         | 1/50 [00:02<01:52,  2.30s/it]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:   4%|▍         | 2/50 [00:02<00:59,  1.24s/it]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:   6%|▌         | 3/50 [00:03<00:36,  1.28it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:   8%|▊         | 4/50 [00:03<00:26,  1.74it/s]\u001b[A\n",
      "Evaluating workflow:  16%|█▌        | 8/50 [00:03<00:08,  4.81it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  20%|██        | 10/50 [00:03<00:07,  5.00it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  28%|██▊       | 14/50 [00:04<00:05,  6.71it/s]\u001b[A\n",
      "Evaluating workflow:  32%|███▏      | 16/50 [00:04<00:04,  7.67it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  38%|███▊      | 19/50 [00:04<00:03,  8.72it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  42%|████▏     | 21/50 [00:05<00:04,  7.01it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  44%|████▍     | 22/50 [00:05<00:07,  3.85it/s]\u001b[A\n",
      "Evaluating workflow:  46%|████▌     | 23/50 [00:06<00:06,  4.02it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[A\n",
      "Evaluating workflow:  48%|████▊     | 24/50 [00:06<00:06,  4.19it/s]\u001b[A\n",
      "Evaluating workflow:  50%|█████     | 25/50 [00:06<00:05,  4.52it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  52%|█████▏    | 26/50 [00:06<00:05,  4.17it/s]\u001b[A\n",
      "Evaluating workflow:  56%|█████▌    | 28/50 [00:07<00:03,  5.65it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  60%|██████    | 30/50 [00:07<00:03,  6.06it/s]\u001b[A\n",
      "Evaluating workflow:  66%|██████▌   | 33/50 [00:07<00:01,  9.19it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  72%|███████▏  | 36/50 [00:07<00:01,  9.46it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  78%|███████▊  | 39/50 [00:08<00:01,  9.56it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  82%|████████▏ | 41/50 [00:08<00:00,  9.51it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  86%|████████▌ | 43/50 [00:09<00:01,  5.40it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  88%|████████▊ | 44/50 [00:09<00:01,  4.21it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  90%|█████████ | 45/50 [00:10<00:01,  3.13it/s]\u001b[A\n",
      "Evaluating workflow:  94%|█████████▍| 47/50 [00:10<00:00,  4.32it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  96%|█████████▌| 48/50 [00:10<00:00,  4.74it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  98%|█████████▊| 49/50 [00:10<00:00,  3.73it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow: 100%|██████████| 50/50 [00:11<00:00,  4.40it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "\u001b[32m2026-01-04 09:16:48.436\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m275\u001b[0m - \u001b[1mStep 2 metrics: {'f1': 0.0, 'em': 0.0, 'acc': 0.5}\u001b[0m\n",
      "\u001b[32m2026-01-04 09:16:48.437\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m291\u001b[0m - \u001b[1mMetrics are worse than the best snapshot which has {'f1': 0.0, 'em': 0.0, 'acc': 0.52}. Rolling back to the best snapshot.\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "\r",
      " 10%|█         | 2/20 [04:16<38:11, 127.31s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2026-01-04 09:16:48.440\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36mstep\u001b[0m:\u001b[36m319\u001b[0m - \u001b[1mExecuting workflow...\u001b[0m\n",
      "\u001b[32m2026-01-04 09:17:05.126\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36mstep\u001b[0m:\u001b[36m347\u001b[0m - \u001b[1mComputing gradients...\u001b[0m\n",
      "\u001b[32m2026-01-04 09:18:41.361\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36mstep\u001b[0m:\u001b[36m349\u001b[0m - \u001b[1mUpdating agents...\u001b[0m\n",
      "\u001b[32m2026-01-04 09:18:48.212\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36mstep\u001b[0m:\u001b[36m353\u001b[0m - \u001b[1mAgents updated\u001b[0m\n",
      "\u001b[32m2026-01-04 09:18:48.212\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m271\u001b[0m - \u001b[1mEvaluating the workflow at step 3 ...\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:   0%|          | 0/50 [00:00<?, ?it/s]\u001b[A\n",
      "Evaluating workflow:   2%|▏         | 1/50 [00:03<02:28,  3.02s/it]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:   4%|▍         | 2/50 [00:03<01:08,  1.42s/it]\u001b[A\n",
      "Evaluating workflow:   6%|▌         | 3/50 [00:03<00:39,  1.19it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:   8%|▊         | 4/50 [00:03<00:27,  1.68it/s]\u001b[A\n",
      "Evaluating workflow:  14%|█▍        | 7/50 [00:03<00:11,  3.88it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  16%|█▌        | 8/50 [00:04<00:09,  4.35it/s]\u001b[A\n",
      "Evaluating workflow:  24%|██▍       | 12/50 [00:04<00:04,  8.50it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  34%|███▍      | 17/50 [00:04<00:03, 10.78it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  38%|███▊      | 19/50 [00:04<00:03,  8.20it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  42%|████▏     | 21/50 [00:06<00:09,  3.02it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  46%|████▌     | 23/50 [00:07<00:07,  3.53it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  48%|████▊     | 24/50 [00:07<00:07,  3.26it/s]\u001b[A\n",
      "Evaluating workflow:  54%|█████▍    | 27/50 [00:07<00:04,  4.88it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  58%|█████▊    | 29/50 [00:07<00:03,  6.05it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  62%|██████▏   | 31/50 [00:08<00:02,  6.89it/s]\u001b[A\n",
      "Evaluating workflow:  68%|██████▊   | 34/50 [00:08<00:01,  8.97it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  72%|███████▏  | 36/50 [00:09<00:02,  5.44it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  76%|███████▌  | 38/50 [00:10<00:03,  3.56it/s]\u001b[A\n",
      "Evaluating workflow:  78%|███████▊  | 39/50 [00:10<00:02,  3.92it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  80%|████████  | 40/50 [00:10<00:02,  4.30it/s]\u001b[A\n",
      "Evaluating workflow:  82%|████████▏ | 41/50 [00:10<00:02,  4.47it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  86%|████████▌ | 43/50 [00:10<00:01,  4.71it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  90%|█████████ | 45/50 [00:11<00:01,  4.39it/s]\u001b[A\n",
      "Evaluating workflow:  94%|█████████▍| 47/50 [00:11<00:00,  5.83it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  96%|█████████▌| 48/50 [00:11<00:00,  5.38it/s]\u001b[A\n",
      "Evaluating workflow:  98%|█████████▊| 49/50 [00:11<00:00,  5.56it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow: 100%|██████████| 50/50 [00:12<00:00,  3.92it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "\u001b[32m2026-01-04 09:19:01.092\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m275\u001b[0m - \u001b[1mStep 3 metrics: {'f1': 0.0, 'em': 0.0, 'acc': 0.46}\u001b[0m\n",
      "\u001b[32m2026-01-04 09:19:01.092\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m291\u001b[0m - \u001b[1mMetrics are worse than the best snapshot which has {'f1': 0.0, 'em': 0.0, 'acc': 0.52}. Rolling back to the best snapshot.\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "\r",
      " 15%|█▌        | 3/20 [06:29<36:45, 129.75s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2026-01-04 09:19:01.096\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36mstep\u001b[0m:\u001b[36m319\u001b[0m - \u001b[1mExecuting workflow...\u001b[0m\n",
      "\u001b[32m2026-01-04 09:19:15.434\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36mstep\u001b[0m:\u001b[36m347\u001b[0m - \u001b[1mComputing gradients...\u001b[0m\n",
      "\u001b[32m2026-01-04 09:21:03.359\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36mstep\u001b[0m:\u001b[36m349\u001b[0m - \u001b[1mUpdating agents...\u001b[0m\n",
      "\u001b[32m2026-01-04 09:21:09.382\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36mstep\u001b[0m:\u001b[36m353\u001b[0m - \u001b[1mAgents updated\u001b[0m\n",
      "\u001b[32m2026-01-04 09:21:09.383\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m271\u001b[0m - \u001b[1mEvaluating the workflow at step 4 ...\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:   0%|          | 0/50 [00:00<?, ?it/s]\u001b[A\n",
      "Evaluating workflow:   2%|▏         | 1/50 [00:03<02:36,  3.19s/it]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:   4%|▍         | 2/50 [00:03<01:12,  1.51s/it]\u001b[A\n",
      "Evaluating workflow:   8%|▊         | 4/50 [00:03<00:27,  1.68it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  16%|█▌        | 8/50 [00:04<00:11,  3.60it/s]\u001b[A\n",
      "Evaluating workflow:  20%|██        | 10/50 [00:04<00:08,  4.69it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  24%|██▍       | 12/50 [00:04<00:07,  5.20it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  30%|███       | 15/50 [00:04<00:05,  6.66it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  34%|███▍      | 17/50 [00:04<00:04,  6.93it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  36%|███▌      | 18/50 [00:05<00:05,  5.37it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  38%|███▊      | 19/50 [00:06<00:11,  2.70it/s]\u001b[A\n",
      "Evaluating workflow:  40%|████      | 20/50 [00:06<00:09,  3.02it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  42%|████▏     | 21/50 [00:07<00:10,  2.67it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  44%|████▍     | 22/50 [00:07<00:10,  2.63it/s]\u001b[A\n",
      "Evaluating workflow:  46%|████▌     | 23/50 [00:07<00:08,  3.26it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  50%|█████     | 25/50 [00:07<00:05,  4.57it/s]\u001b[A\n",
      "Evaluating workflow:  56%|█████▌    | 28/50 [00:08<00:03,  7.19it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  60%|██████    | 30/50 [00:08<00:02,  8.44it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  66%|██████▌   | 33/50 [00:09<00:03,  5.32it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  78%|███████▊  | 39/50 [00:09<00:01,  7.11it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  80%|████████  | 40/50 [00:11<00:02,  3.66it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  82%|████████▏ | 41/50 [00:11<00:02,  3.77it/s]\u001b[A\n",
      "Evaluating workflow:  86%|████████▌ | 43/50 [00:11<00:01,  4.78it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  88%|████████▊ | 44/50 [00:11<00:01,  4.96it/s]\u001b[A\n",
      "Evaluating workflow:  92%|█████████▏| 46/50 [00:11<00:00,  6.05it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  94%|█████████▍| 47/50 [00:11<00:00,  5.95it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  96%|█████████▌| 48/50 [00:12<00:00,  5.71it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow: 100%|██████████| 50/50 [00:12<00:00,  3.87it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "\u001b[32m2026-01-04 09:21:22.442\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m275\u001b[0m - \u001b[1mStep 4 metrics: {'f1': 0.0, 'em': 0.0, 'acc': 0.52}\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "\r",
      " 20%|██        | 4/20 [08:50<35:49, 134.33s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2026-01-04 09:21:22.444\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36mstep\u001b[0m:\u001b[36m319\u001b[0m - \u001b[1mExecuting workflow...\u001b[0m\n",
      "\u001b[32m2026-01-04 09:21:40.585\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36mstep\u001b[0m:\u001b[36m347\u001b[0m - \u001b[1mComputing gradients...\u001b[0m\n",
      "\u001b[32m2026-01-04 09:23:19.276\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36mstep\u001b[0m:\u001b[36m349\u001b[0m - \u001b[1mUpdating agents...\u001b[0m\n",
      "\u001b[32m2026-01-04 09:23:25.725\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36mstep\u001b[0m:\u001b[36m353\u001b[0m - \u001b[1mAgents updated\u001b[0m\n",
      "\u001b[32m2026-01-04 09:23:25.726\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m271\u001b[0m - \u001b[1mEvaluating the workflow at step 5 ...\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:   0%|          | 0/50 [00:00<?, ?it/s]\u001b[A\n",
      "Evaluating workflow:   2%|▏         | 1/50 [00:02<02:12,  2.70s/it]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:   6%|▌         | 3/50 [00:03<00:39,  1.18it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:   8%|▊         | 4/50 [00:03<00:31,  1.44it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  10%|█         | 5/50 [00:03<00:25,  1.75it/s]\u001b[A\n",
      "Evaluating workflow:  12%|█▏        | 6/50 [00:04<00:19,  2.24it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  16%|█▌        | 8/50 [00:04<00:10,  3.88it/s]\u001b[A\n",
      "Evaluating workflow:  20%|██        | 10/50 [00:04<00:07,  5.56it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  24%|██▍       | 12/50 [00:04<00:05,  6.83it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  32%|███▏      | 16/50 [00:04<00:03, 11.17it/s]\u001b[A\n",
      "Evaluating workflow:  38%|███▊      | 19/50 [00:04<00:02, 12.39it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  42%|████▏     | 21/50 [00:06<00:09,  3.14it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  46%|████▌     | 23/50 [00:07<00:08,  3.26it/s]\u001b[A\n",
      "Evaluating workflow:  48%|████▊     | 24/50 [00:07<00:07,  3.61it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  50%|█████     | 25/50 [00:08<00:08,  3.02it/s]\u001b[A\n",
      "Evaluating workflow:  52%|█████▏    | 26/50 [00:08<00:07,  3.39it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  54%|█████▍    | 27/50 [00:08<00:07,  3.22it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  68%|██████▊   | 34/50 [00:09<00:02,  7.12it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  78%|███████▊  | 39/50 [00:10<00:01,  5.75it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  80%|████████  | 40/50 [00:11<00:02,  3.50it/s]\u001b[A\n",
      "Evaluating workflow:  86%|████████▌ | 43/50 [00:11<00:01,  4.71it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  88%|████████▊ | 44/50 [00:11<00:01,  4.42it/s]\u001b[A\n",
      "Evaluating workflow:  90%|█████████ | 45/50 [00:11<00:01,  4.71it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  92%|█████████▏| 46/50 [00:12<00:01,  3.81it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  94%|█████████▍| 47/50 [00:12<00:00,  3.39it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  98%|█████████▊| 49/50 [00:13<00:00,  3.71it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow: 100%|██████████| 50/50 [00:15<00:00,  3.24it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "\u001b[32m2026-01-04 09:23:41.286\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m275\u001b[0m - \u001b[1mStep 5 metrics: {'f1': 0.0, 'em': 0.0, 'acc': 0.36}\u001b[0m\n",
      "\u001b[32m2026-01-04 09:23:41.286\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m291\u001b[0m - \u001b[1mMetrics are worse than the best snapshot which has {'f1': 0.0, 'em': 0.0, 'acc': 0.52}. Rolling back to the best snapshot.\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "\r",
      " 25%|██▌       | 5/20 [11:09<33:59, 135.96s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2026-01-04 09:23:41.290\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36mstep\u001b[0m:\u001b[36m319\u001b[0m - \u001b[1mExecuting workflow...\u001b[0m\n",
      "\u001b[32m2026-01-04 09:23:58.831\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36mstep\u001b[0m:\u001b[36m347\u001b[0m - \u001b[1mComputing gradients...\u001b[0m\n",
      "\u001b[32m2026-01-04 09:25:29.119\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36mstep\u001b[0m:\u001b[36m349\u001b[0m - \u001b[1mUpdating agents...\u001b[0m\n",
      "\u001b[32m2026-01-04 09:25:34.854\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36mstep\u001b[0m:\u001b[36m353\u001b[0m - \u001b[1mAgents updated\u001b[0m\n",
      "\u001b[32m2026-01-04 09:25:34.854\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m271\u001b[0m - \u001b[1mEvaluating the workflow at step 6 ...\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:   0%|          | 0/50 [00:00<?, ?it/s]\u001b[A\n",
      "Evaluating workflow:   2%|▏         | 1/50 [00:02<01:41,  2.07s/it]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:   4%|▍         | 2/50 [00:02<01:01,  1.29s/it]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:   6%|▌         | 3/50 [00:03<00:37,  1.26it/s]\u001b[A\n",
      "Evaluating workflow:  10%|█         | 5/50 [00:03<00:16,  2.67it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  12%|█▏        | 6/50 [00:03<00:14,  3.14it/s]\u001b[A\n",
      "Evaluating workflow:  14%|█▍        | 7/50 [00:03<00:11,  3.66it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  16%|█▌        | 8/50 [00:03<00:10,  3.83it/s]\u001b[A\n",
      "Evaluating workflow:  26%|██▌       | 13/50 [00:03<00:03,  9.98it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  30%|███       | 15/50 [00:04<00:04,  7.78it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  34%|███▍      | 17/50 [00:04<00:04,  7.58it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  38%|███▊      | 19/50 [00:04<00:03,  7.90it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  42%|████▏     | 21/50 [00:05<00:04,  6.20it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  44%|████▍     | 22/50 [00:05<00:07,  3.98it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  46%|████▌     | 23/50 [00:06<00:06,  4.04it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  48%|████▊     | 24/50 [00:06<00:07,  3.31it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  50%|█████     | 25/50 [00:06<00:06,  3.59it/s]\u001b[A\n",
      "Evaluating workflow:  52%|█████▏    | 26/50 [00:06<00:05,  4.16it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  54%|█████▍    | 27/50 [00:07<00:05,  4.34it/s]\u001b[A\n",
      "Evaluating workflow:  56%|█████▌    | 28/50 [00:07<00:04,  4.84it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  60%|██████    | 30/50 [00:07<00:03,  6.16it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  64%|██████▍   | 32/50 [00:07<00:02,  6.31it/s]\u001b[A\n",
      "Evaluating workflow:  66%|██████▌   | 33/50 [00:07<00:02,  6.39it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  68%|██████▊   | 34/50 [00:08<00:02,  6.43it/s]\u001b[A\n",
      "Evaluating workflow:  76%|███████▌  | 38/50 [00:08<00:01, 11.31it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  80%|████████  | 40/50 [00:08<00:01,  7.53it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  82%|████████▏ | 41/50 [00:08<00:01,  6.92it/s]\u001b[A\n",
      "Evaluating workflow:  84%|████████▍ | 42/50 [00:09<00:01,  7.19it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  86%|████████▌ | 43/50 [00:09<00:01,  5.89it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  88%|████████▊ | 44/50 [00:10<00:01,  3.48it/s]\u001b[A\n",
      "Evaluating workflow:  90%|█████████ | 45/50 [00:10<00:01,  3.77it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  94%|█████████▍| 47/50 [00:10<00:00,  4.50it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  96%|█████████▌| 48/50 [00:10<00:00,  4.16it/s]\u001b[A\n",
      "Evaluating workflow:  98%|█████████▊| 49/50 [00:11<00:00,  4.42it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow: 100%|██████████| 50/50 [00:11<00:00,  4.52it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "\u001b[32m2026-01-04 09:25:46.034\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m275\u001b[0m - \u001b[1mStep 6 metrics: {'f1': 0.0, 'em': 0.0, 'acc': 0.52}\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "\r",
      " 30%|███       | 6/20 [13:14<30:50, 132.15s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2026-01-04 09:25:46.035\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36mstep\u001b[0m:\u001b[36m319\u001b[0m - \u001b[1mExecuting workflow...\u001b[0m\n",
      "\u001b[32m2026-01-04 09:26:05.560\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36mstep\u001b[0m:\u001b[36m347\u001b[0m - \u001b[1mComputing gradients...\u001b[0m\n",
      "\u001b[32m2026-01-04 09:27:48.869\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36mstep\u001b[0m:\u001b[36m349\u001b[0m - \u001b[1mUpdating agents...\u001b[0m\n",
      "\u001b[32m2026-01-04 09:27:55.515\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36mstep\u001b[0m:\u001b[36m353\u001b[0m - \u001b[1mAgents updated\u001b[0m\n",
      "\u001b[32m2026-01-04 09:27:55.516\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m271\u001b[0m - \u001b[1mEvaluating the workflow at step 7 ...\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:   0%|          | 0/50 [00:00<?, ?it/s]\u001b[A\n",
      "Evaluating workflow:   2%|▏         | 1/50 [00:01<01:14,  1.53s/it]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:   4%|▍         | 2/50 [00:02<00:43,  1.10it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:   8%|▊         | 4/50 [00:02<00:26,  1.73it/s]\u001b[A\n",
      "Evaluating workflow:  10%|█         | 5/50 [00:02<00:20,  2.20it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  12%|█▏        | 6/50 [00:03<00:18,  2.37it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  16%|█▌        | 8/50 [00:03<00:11,  3.58it/s]\u001b[A\n",
      "Evaluating workflow:  18%|█▊        | 9/50 [00:03<00:10,  3.95it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  22%|██▏       | 11/50 [00:03<00:06,  5.88it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  28%|██▊       | 14/50 [00:04<00:04,  8.08it/s]\u001b[A\n",
      "Evaluating workflow:  32%|███▏      | 16/50 [00:04<00:03,  9.76it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  36%|███▌      | 18/50 [00:04<00:03,  9.43it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  40%|████      | 20/50 [00:04<00:04,  7.10it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  44%|████▍     | 22/50 [00:05<00:05,  5.03it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  46%|████▌     | 23/50 [00:06<00:07,  3.44it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  48%|████▊     | 24/50 [00:06<00:08,  2.95it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  50%|█████     | 25/50 [00:07<00:09,  2.68it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  52%|█████▏    | 26/50 [00:07<00:07,  3.00it/s]\u001b[A\n",
      "Evaluating workflow:  54%|█████▍    | 27/50 [00:07<00:06,  3.49it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  60%|██████    | 30/50 [00:07<00:03,  6.12it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  62%|██████▏   | 31/50 [00:07<00:03,  5.42it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  64%|██████▍   | 32/50 [00:08<00:05,  3.29it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  72%|███████▏  | 36/50 [00:09<00:02,  5.39it/s]\u001b[A\n",
      "Evaluating workflow:  76%|███████▌  | 38/50 [00:09<00:01,  6.74it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  80%|████████  | 40/50 [00:09<00:01,  5.86it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  82%|████████▏ | 41/50 [00:09<00:01,  5.87it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  84%|████████▍ | 42/50 [00:10<00:02,  3.31it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  88%|████████▊ | 44/50 [00:11<00:02,  2.68it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  92%|█████████▏| 46/50 [00:11<00:01,  3.55it/s]\u001b[A\n",
      "Evaluating workflow:  94%|█████████▍| 47/50 [00:11<00:00,  4.05it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  96%|█████████▌| 48/50 [00:12<00:00,  2.47it/s]\u001b[A\n",
      "Evaluating workflow:  98%|█████████▊| 49/50 [00:13<00:00,  2.97it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow: 100%|██████████| 50/50 [00:15<00:00,  3.29it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "\u001b[32m2026-01-04 09:28:11.213\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m275\u001b[0m - \u001b[1mStep 7 metrics: {'f1': 0.0, 'em': 0.0, 'acc': 0.56}\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "\r",
      " 35%|███▌      | 7/20 [15:39<29:33, 136.41s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2026-01-04 09:28:11.213\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36mstep\u001b[0m:\u001b[36m319\u001b[0m - \u001b[1mExecuting workflow...\u001b[0m\n",
      "\u001b[32m2026-01-04 09:28:32.301\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36mstep\u001b[0m:\u001b[36m347\u001b[0m - \u001b[1mComputing gradients...\u001b[0m\n",
      "\u001b[32m2026-01-04 09:30:21.838\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36mstep\u001b[0m:\u001b[36m349\u001b[0m - \u001b[1mUpdating agents...\u001b[0m\n",
      "\u001b[32m2026-01-04 09:30:28.131\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36mstep\u001b[0m:\u001b[36m353\u001b[0m - \u001b[1mAgents updated\u001b[0m\n",
      "\u001b[32m2026-01-04 09:30:28.131\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m271\u001b[0m - \u001b[1mEvaluating the workflow at step 8 ...\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:   0%|          | 0/50 [00:00<?, ?it/s]\u001b[A\n",
      "Evaluating workflow:   2%|▏         | 1/50 [00:01<01:27,  1.79s/it]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:   4%|▍         | 2/50 [00:02<01:05,  1.37s/it]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:   6%|▌         | 3/50 [00:03<00:46,  1.02it/s]\u001b[A\n",
      "Evaluating workflow:  10%|█         | 5/50 [00:03<00:20,  2.19it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  14%|█▍        | 7/50 [00:03<00:13,  3.20it/s]\u001b[A\n",
      "Evaluating workflow:  20%|██        | 10/50 [00:03<00:07,  5.53it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  24%|██▍       | 12/50 [00:04<00:05,  7.07it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  28%|██▊       | 14/50 [00:04<00:06,  5.73it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  34%|███▍      | 17/50 [00:04<00:04,  8.01it/s]\u001b[A\n",
      "Evaluating workflow:  40%|████      | 20/50 [00:04<00:02, 10.64it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  44%|████▍     | 22/50 [00:05<00:04,  6.04it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  48%|████▊     | 24/50 [00:06<00:06,  4.30it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  50%|█████     | 25/50 [00:06<00:07,  3.56it/s]\u001b[A\n",
      "Evaluating workflow:  54%|█████▍    | 27/50 [00:07<00:05,  4.54it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  56%|█████▌    | 28/50 [00:07<00:04,  4.79it/s]\u001b[A\n",
      "Evaluating workflow:  60%|██████    | 30/50 [00:07<00:03,  6.31it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  64%|██████▍   | 32/50 [00:07<00:03,  5.60it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  74%|███████▍  | 37/50 [00:08<00:01,  8.78it/s]\u001b[A\n",
      "Evaluating workflow:  78%|███████▊  | 39/50 [00:08<00:01,  9.06it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  82%|████████▏ | 41/50 [00:08<00:00,  9.52it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  86%|████████▌ | 43/50 [00:09<00:01,  5.71it/s]\u001b[A\n",
      "Evaluating workflow:  88%|████████▊ | 44/50 [00:09<00:01,  5.95it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  90%|█████████ | 45/50 [00:10<00:01,  2.59it/s]\u001b[A\n",
      "Evaluating workflow:  94%|█████████▍| 47/50 [00:10<00:00,  3.50it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  96%|█████████▌| 48/50 [00:11<00:00,  3.60it/s]\u001b[A\n",
      "Evaluating workflow: 100%|██████████| 50/50 [00:11<00:00,  4.44it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "\u001b[32m2026-01-04 09:30:39.530\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m275\u001b[0m - \u001b[1mStep 8 metrics: {'f1': 0.0, 'em': 0.0, 'acc': 0.62}\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "\r",
      " 40%|████      | 8/20 [18:07<28:02, 140.20s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2026-01-04 09:30:39.531\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36mstep\u001b[0m:\u001b[36m319\u001b[0m - \u001b[1mExecuting workflow...\u001b[0m\n",
      "\u001b[32m2026-01-04 09:31:01.485\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36mstep\u001b[0m:\u001b[36m347\u001b[0m - \u001b[1mComputing gradients...\u001b[0m\n",
      "\u001b[32m2026-01-04 09:32:49.323\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36mstep\u001b[0m:\u001b[36m349\u001b[0m - \u001b[1mUpdating agents...\u001b[0m\n",
      "\u001b[32m2026-01-04 09:32:56.077\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36mstep\u001b[0m:\u001b[36m353\u001b[0m - \u001b[1mAgents updated\u001b[0m\n",
      "\u001b[32m2026-01-04 09:32:56.077\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m271\u001b[0m - \u001b[1mEvaluating the workflow at step 9 ...\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:   0%|          | 0/50 [00:00<?, ?it/s]\u001b[A\n",
      "Evaluating workflow:   2%|▏         | 1/50 [00:01<01:25,  1.74s/it]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:   6%|▌         | 3/50 [00:02<00:34,  1.36it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:   8%|▊         | 4/50 [00:03<00:30,  1.49it/s]\u001b[A\n",
      "Evaluating workflow:  12%|█▏        | 6/50 [00:03<00:16,  2.64it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  20%|██        | 10/50 [00:03<00:07,  5.26it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  26%|██▌       | 13/50 [00:03<00:04,  7.66it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  32%|███▏      | 16/50 [00:03<00:03,  8.99it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  36%|███▌      | 18/50 [00:04<00:05,  5.37it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  40%|████      | 20/50 [00:04<00:05,  5.67it/s]\u001b[A\n",
      "Evaluating workflow:  46%|████▌     | 23/50 [00:05<00:03,  7.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[A\n",
      "Evaluating workflow:  50%|█████     | 25/50 [00:06<00:06,  3.67it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  54%|█████▍    | 27/50 [00:06<00:05,  4.09it/s]\u001b[A\n",
      "Evaluating workflow:  56%|█████▌    | 28/50 [00:06<00:05,  4.33it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  60%|██████    | 30/50 [00:07<00:03,  5.43it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  66%|██████▌   | 33/50 [00:07<00:02,  7.14it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  72%|███████▏  | 36/50 [00:07<00:01,  7.99it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  76%|███████▌  | 38/50 [00:08<00:01,  6.90it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  78%|███████▊  | 39/50 [00:08<00:01,  6.24it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  82%|████████▏ | 41/50 [00:08<00:01,  6.82it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  84%|████████▍ | 42/50 [00:08<00:01,  4.78it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  86%|████████▌ | 43/50 [00:09<00:02,  3.42it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  90%|█████████ | 45/50 [00:10<00:01,  2.97it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  96%|█████████▌| 48/50 [00:10<00:00,  4.27it/s]\u001b[A\n",
      "Evaluating workflow:  98%|█████████▊| 49/50 [00:10<00:00,  4.64it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow: 100%|██████████| 50/50 [00:11<00:00,  4.39it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "\u001b[32m2026-01-04 09:33:07.598\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m275\u001b[0m - \u001b[1mStep 9 metrics: {'f1': 0.0, 'em': 0.0, 'acc': 0.5}\u001b[0m\n",
      "\u001b[32m2026-01-04 09:33:07.598\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m291\u001b[0m - \u001b[1mMetrics are worse than the best snapshot which has {'f1': 0.0, 'em': 0.0, 'acc': 0.62}. Rolling back to the best snapshot.\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "\r",
      " 45%|████▌     | 9/20 [20:35<26:09, 142.66s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2026-01-04 09:33:07.602\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36mstep\u001b[0m:\u001b[36m319\u001b[0m - \u001b[1mExecuting workflow...\u001b[0m\n",
      "\u001b[32m2026-01-04 09:33:23.457\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36mstep\u001b[0m:\u001b[36m347\u001b[0m - \u001b[1mComputing gradients...\u001b[0m\n",
      "\u001b[32m2026-01-04 09:34:53.320\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36mstep\u001b[0m:\u001b[36m349\u001b[0m - \u001b[1mUpdating agents...\u001b[0m\n",
      "\u001b[32m2026-01-04 09:34:59.334\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36mstep\u001b[0m:\u001b[36m353\u001b[0m - \u001b[1mAgents updated\u001b[0m\n",
      "\u001b[32m2026-01-04 09:34:59.334\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m271\u001b[0m - \u001b[1mEvaluating the workflow at step 10 ...\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:   0%|          | 0/50 [00:00<?, ?it/s]\u001b[A\n",
      "Evaluating workflow:   2%|▏         | 1/50 [00:02<01:40,  2.04s/it]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:   4%|▍         | 2/50 [00:02<00:48,  1.01s/it]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:   8%|▊         | 4/50 [00:03<00:29,  1.54it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  10%|█         | 5/50 [00:03<00:23,  1.93it/s]\u001b[A\n",
      "Evaluating workflow:  14%|█▍        | 7/50 [00:03<00:13,  3.16it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  16%|█▌        | 8/50 [00:03<00:11,  3.67it/s]\u001b[A\n",
      "Evaluating workflow:  20%|██        | 10/50 [00:03<00:07,  5.51it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  28%|██▊       | 14/50 [00:04<00:05,  7.19it/s]\u001b[A\n",
      "Evaluating workflow:  44%|████▍     | 22/50 [00:04<00:01, 15.83it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  50%|█████     | 25/50 [00:06<00:05,  4.69it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  56%|█████▌    | 28/50 [00:07<00:04,  4.76it/s]\u001b[A\n",
      "Evaluating workflow:  62%|██████▏   | 31/50 [00:07<00:03,  6.16it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  66%|██████▌   | 33/50 [00:07<00:02,  6.88it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  70%|███████   | 35/50 [00:07<00:02,  7.31it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  74%|███████▍  | 37/50 [00:07<00:01,  7.09it/s]\u001b[A\n",
      "Evaluating workflow:  78%|███████▊  | 39/50 [00:07<00:01,  8.35it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  82%|████████▏ | 41/50 [00:08<00:01,  7.95it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  86%|████████▌ | 43/50 [00:08<00:01,  6.92it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  90%|█████████ | 45/50 [00:08<00:00,  7.03it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  92%|█████████▏| 46/50 [00:09<00:00,  4.45it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  94%|█████████▍| 47/50 [00:09<00:00,  4.36it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  96%|█████████▌| 48/50 [00:10<00:00,  3.98it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  98%|█████████▊| 49/50 [00:10<00:00,  2.90it/s]\u001b[A\n",
      "Evaluating workflow: 100%|██████████| 50/50 [00:10<00:00,  4.61it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "\u001b[32m2026-01-04 09:35:10.311\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m275\u001b[0m - \u001b[1mStep 10 metrics: {'f1': 0.0, 'em': 0.0, 'acc': 0.62}\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "\r",
      " 50%|█████     | 10/20 [22:38<22:45, 136.50s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2026-01-04 09:35:10.312\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36mstep\u001b[0m:\u001b[36m319\u001b[0m - \u001b[1mExecuting workflow...\u001b[0m\n",
      "\u001b[32m2026-01-04 09:35:28.050\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36mstep\u001b[0m:\u001b[36m347\u001b[0m - \u001b[1mComputing gradients...\u001b[0m\n",
      "\u001b[32m2026-01-04 09:36:52.372\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36mstep\u001b[0m:\u001b[36m349\u001b[0m - \u001b[1mUpdating agents...\u001b[0m\n",
      "\u001b[32m2026-01-04 09:36:56.523\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36mstep\u001b[0m:\u001b[36m353\u001b[0m - \u001b[1mAgents updated\u001b[0m\n",
      "\u001b[32m2026-01-04 09:36:56.523\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m271\u001b[0m - \u001b[1mEvaluating the workflow at step 11 ...\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:   0%|          | 0/50 [00:00<?, ?it/s]\u001b[A\n",
      "Evaluating workflow:   2%|▏         | 1/50 [00:01<01:35,  1.95s/it]\u001b[A\n",
      "Evaluating workflow:   4%|▍         | 2/50 [00:02<00:43,  1.11it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  10%|█         | 5/50 [00:02<00:13,  3.30it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  12%|█▏        | 6/50 [00:02<00:13,  3.24it/s]\u001b[A\n",
      "Evaluating workflow:  18%|█▊        | 9/50 [00:02<00:06,  5.95it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  22%|██▏       | 11/50 [00:02<00:05,  7.68it/s]\u001b[A\n",
      "Evaluating workflow:  28%|██▊       | 14/50 [00:03<00:03,  9.68it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  32%|███▏      | 16/50 [00:03<00:03, 10.71it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  36%|███▌      | 18/50 [00:03<00:03,  9.89it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  40%|████      | 20/50 [00:03<00:02, 10.70it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  44%|████▍     | 22/50 [00:04<00:05,  5.55it/s]\u001b[A\n",
      "Evaluating workflow:  48%|████▊     | 24/50 [00:04<00:03,  6.93it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  52%|█████▏    | 26/50 [00:05<00:05,  4.57it/s]\u001b[A\n",
      "Evaluating workflow:  54%|█████▍    | 27/50 [00:05<00:04,  4.87it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  58%|█████▊    | 29/50 [00:05<00:04,  5.23it/s]\u001b[A\n",
      "Evaluating workflow:  62%|██████▏   | 31/50 [00:05<00:02,  6.69it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  68%|██████▊   | 34/50 [00:06<00:01,  9.01it/s]\u001b[A\n",
      "Evaluating workflow:  72%|███████▏  | 36/50 [00:06<00:01,  9.66it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  76%|███████▌  | 38/50 [00:06<00:01,  9.03it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  84%|████████▍ | 42/50 [00:06<00:00, 10.73it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  88%|████████▊ | 44/50 [00:07<00:00,  7.03it/s]\u001b[A\n",
      "Evaluating workflow:  90%|█████████ | 45/50 [00:07<00:00,  7.27it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  92%|█████████▏| 46/50 [00:07<00:00,  6.24it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  96%|█████████▌| 48/50 [00:08<00:00,  4.58it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow: 100%|██████████| 50/50 [00:09<00:00,  5.32it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "\u001b[32m2026-01-04 09:37:06.056\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m275\u001b[0m - \u001b[1mStep 11 metrics: {'f1': 0.0, 'em': 0.0, 'acc': 0.64}\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "\r",
      " 55%|█████▌    | 11/20 [24:34<19:31, 130.15s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2026-01-04 09:37:06.057\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36mstep\u001b[0m:\u001b[36m319\u001b[0m - \u001b[1mExecuting workflow...\u001b[0m\n",
      "\u001b[32m2026-01-04 09:37:20.400\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36mstep\u001b[0m:\u001b[36m347\u001b[0m - \u001b[1mComputing gradients...\u001b[0m\n",
      "\u001b[32m2026-01-04 09:38:42.445\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36mstep\u001b[0m:\u001b[36m349\u001b[0m - \u001b[1mUpdating agents...\u001b[0m\n",
      "\u001b[32m2026-01-04 09:38:47.350\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36mstep\u001b[0m:\u001b[36m353\u001b[0m - \u001b[1mAgents updated\u001b[0m\n",
      "\u001b[32m2026-01-04 09:38:47.350\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m271\u001b[0m - \u001b[1mEvaluating the workflow at step 12 ...\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:   0%|          | 0/50 [00:00<?, ?it/s]\u001b[A\n",
      "Evaluating workflow:   2%|▏         | 1/50 [00:01<01:17,  1.59s/it]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:   6%|▌         | 3/50 [00:01<00:24,  1.91it/s]\u001b[A\n",
      "Evaluating workflow:  10%|█         | 5/50 [00:02<00:13,  3.41it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  12%|█▏        | 6/50 [00:02<00:15,  2.84it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  14%|█▍        | 7/50 [00:02<00:15,  2.75it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  26%|██▌       | 13/50 [00:03<00:05,  7.20it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  32%|███▏      | 16/50 [00:03<00:04,  7.39it/s]\u001b[A\n",
      "Evaluating workflow:  36%|███▌      | 18/50 [00:03<00:03,  8.65it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  40%|████      | 20/50 [00:03<00:03,  8.99it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  44%|████▍     | 22/50 [00:04<00:03,  7.29it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  46%|████▌     | 23/50 [00:04<00:04,  6.59it/s]\u001b[A\n",
      "Evaluating workflow:  50%|█████     | 25/50 [00:04<00:03,  8.03it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  54%|█████▍    | 27/50 [00:05<00:04,  4.99it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  58%|█████▊    | 29/50 [00:05<00:04,  5.00it/s]\u001b[A\n",
      "Evaluating workflow:  60%|██████    | 30/50 [00:05<00:03,  5.13it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  62%|██████▏   | 31/50 [00:06<00:03,  5.21it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  66%|██████▌   | 33/50 [00:06<00:02,  5.67it/s]\u001b[A\n",
      "Evaluating workflow:  70%|███████   | 35/50 [00:06<00:02,  6.69it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  72%|███████▏  | 36/50 [00:07<00:02,  5.13it/s]\u001b[A\n",
      "Evaluating workflow:  74%|███████▍  | 37/50 [00:07<00:02,  5.63it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  80%|████████  | 40/50 [00:07<00:01,  8.15it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  82%|████████▏ | 41/50 [00:07<00:01,  7.00it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  90%|█████████ | 45/50 [00:08<00:00,  6.85it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow: 100%|██████████| 50/50 [00:08<00:00,  5.62it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "\u001b[32m2026-01-04 09:38:56.372\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m275\u001b[0m - \u001b[1mStep 12 metrics: {'f1': 0.0, 'em': 0.0, 'acc': 0.56}\u001b[0m\n",
      "\u001b[32m2026-01-04 09:38:56.372\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m291\u001b[0m - \u001b[1mMetrics are worse than the best snapshot which has {'f1': 0.0, 'em': 0.0, 'acc': 0.64}. Rolling back to the best snapshot.\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "\r",
      " 60%|██████    | 12/20 [26:24<16:32, 124.12s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2026-01-04 09:38:56.376\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36mstep\u001b[0m:\u001b[36m319\u001b[0m - \u001b[1mExecuting workflow...\u001b[0m\n",
      "\u001b[32m2026-01-04 09:39:16.788\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36mstep\u001b[0m:\u001b[36m347\u001b[0m - \u001b[1mComputing gradients...\u001b[0m\n",
      "\u001b[32m2026-01-04 09:40:54.728\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36mstep\u001b[0m:\u001b[36m349\u001b[0m - \u001b[1mUpdating agents...\u001b[0m\n",
      "\u001b[32m2026-01-04 09:40:59.634\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36mstep\u001b[0m:\u001b[36m353\u001b[0m - \u001b[1mAgents updated\u001b[0m\n",
      "\u001b[32m2026-01-04 09:40:59.634\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m271\u001b[0m - \u001b[1mEvaluating the workflow at step 13 ...\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:   0%|          | 0/50 [00:00<?, ?it/s]\u001b[A\n",
      "Evaluating workflow:   2%|▏         | 1/50 [00:01<01:23,  1.71s/it]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:   6%|▌         | 3/50 [00:02<00:27,  1.69it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:   8%|▊         | 4/50 [00:02<00:23,  1.99it/s]\u001b[A\n",
      "Evaluating workflow:  16%|█▌        | 8/50 [00:02<00:07,  5.27it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  20%|██        | 10/50 [00:03<00:08,  4.74it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  28%|██▊       | 14/50 [00:03<00:05,  6.98it/s]\u001b[A\n",
      "Evaluating workflow:  34%|███▍      | 17/50 [00:03<00:03,  9.37it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  40%|████      | 20/50 [00:03<00:02, 11.49it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  44%|████▍     | 22/50 [00:04<00:03,  8.68it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  48%|████▊     | 24/50 [00:04<00:03,  7.89it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  52%|█████▏    | 26/50 [00:04<00:03,  7.34it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  56%|█████▌    | 28/50 [00:05<00:05,  3.70it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  58%|█████▊    | 29/50 [00:06<00:05,  3.81it/s]\u001b[A\n",
      "Evaluating workflow:  66%|██████▌   | 33/50 [00:06<00:02,  6.36it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  70%|███████   | 35/50 [00:06<00:02,  7.03it/s]\u001b[A\n",
      "Evaluating workflow:  76%|███████▌  | 38/50 [00:06<00:01,  9.49it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  80%|████████  | 40/50 [00:06<00:01,  9.96it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  84%|████████▍ | 42/50 [00:07<00:00,  9.40it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  88%|████████▊ | 44/50 [00:07<00:00,  8.59it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  92%|█████████▏| 46/50 [00:07<00:00,  6.98it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  94%|█████████▍| 47/50 [00:08<00:00,  5.20it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  96%|█████████▌| 48/50 [00:08<00:00,  3.80it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  98%|█████████▊| 49/50 [00:09<00:00,  2.62it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow: 100%|██████████| 50/50 [00:10<00:00,  4.85it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "\u001b[32m2026-01-04 09:41:10.063\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m275\u001b[0m - \u001b[1mStep 13 metrics: {'f1': 0.0, 'em': 0.0, 'acc': 0.74}\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "\r",
      " 65%|██████▌   | 13/20 [28:38<14:49, 127.02s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2026-01-04 09:41:10.064\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36mstep\u001b[0m:\u001b[36m319\u001b[0m - \u001b[1mExecuting workflow...\u001b[0m\n",
      "\u001b[32m2026-01-04 09:41:29.147\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36mstep\u001b[0m:\u001b[36m347\u001b[0m - \u001b[1mComputing gradients...\u001b[0m\n",
      "\u001b[32m2026-01-04 09:43:03.517\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36mstep\u001b[0m:\u001b[36m349\u001b[0m - \u001b[1mUpdating agents...\u001b[0m\n",
      "\u001b[32m2026-01-04 09:43:08.789\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36mstep\u001b[0m:\u001b[36m353\u001b[0m - \u001b[1mAgents updated\u001b[0m\n",
      "\u001b[32m2026-01-04 09:43:08.789\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m271\u001b[0m - \u001b[1mEvaluating the workflow at step 14 ...\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:   0%|          | 0/50 [00:00<?, ?it/s]\u001b[A\n",
      "Evaluating workflow:   2%|▏         | 1/50 [00:01<01:29,  1.84s/it]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:   8%|▊         | 4/50 [00:02<00:19,  2.35it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  10%|█         | 5/50 [00:02<00:18,  2.39it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  12%|█▏        | 6/50 [00:02<00:17,  2.47it/s]\u001b[A\n",
      "Evaluating workflow:  18%|█▊        | 9/50 [00:03<00:08,  4.60it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  24%|██▍       | 12/50 [00:03<00:05,  7.21it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  28%|██▊       | 14/50 [00:03<00:05,  7.11it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  32%|███▏      | 16/50 [00:03<00:04,  7.27it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  36%|███▌      | 18/50 [00:04<00:05,  6.05it/s]\u001b[A\n",
      "Evaluating workflow:  42%|████▏     | 21/50 [00:04<00:03,  8.11it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  46%|████▌     | 23/50 [00:04<00:02,  9.37it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  50%|█████     | 25/50 [00:05<00:05,  4.33it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  52%|█████▏    | 26/50 [00:05<00:05,  4.11it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  58%|█████▊    | 29/50 [00:06<00:03,  5.62it/s]\u001b[A\n",
      "Evaluating workflow:  62%|██████▏   | 31/50 [00:06<00:02,  6.99it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  66%|██████▌   | 33/50 [00:06<00:02,  7.51it/s]\u001b[A\n",
      "Evaluating workflow:  72%|███████▏  | 36/50 [00:06<00:01, 10.38it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  76%|███████▌  | 38/50 [00:07<00:01,  8.04it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  80%|████████  | 40/50 [00:07<00:01,  6.77it/s]\u001b[A\n",
      "Evaluating workflow:  84%|████████▍ | 42/50 [00:07<00:00,  8.16it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  88%|████████▊ | 44/50 [00:08<00:00,  6.30it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  90%|█████████ | 45/50 [00:08<00:00,  5.48it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  96%|█████████▌| 48/50 [00:10<00:00,  2.96it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  98%|█████████▊| 49/50 [00:10<00:00,  2.42it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow: 100%|██████████| 50/50 [00:17<00:00,  2.81it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "\u001b[32m2026-01-04 09:43:26.688\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m275\u001b[0m - \u001b[1mStep 14 metrics: {'f1': 0.0, 'em': 0.0, 'acc': 0.78}\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "\r",
      " 70%|███████   | 14/20 [30:54<12:59, 129.92s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2026-01-04 09:43:26.689\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36mstep\u001b[0m:\u001b[36m319\u001b[0m - \u001b[1mExecuting workflow...\u001b[0m\n",
      "\u001b[32m2026-01-04 09:43:41.500\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36mstep\u001b[0m:\u001b[36m347\u001b[0m - \u001b[1mComputing gradients...\u001b[0m\n",
      "\u001b[32m2026-01-04 09:45:10.706\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36mstep\u001b[0m:\u001b[36m349\u001b[0m - \u001b[1mUpdating agents...\u001b[0m\n",
      "\u001b[32m2026-01-04 09:45:16.346\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36mstep\u001b[0m:\u001b[36m353\u001b[0m - \u001b[1mAgents updated\u001b[0m\n",
      "\u001b[32m2026-01-04 09:45:16.346\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m271\u001b[0m - \u001b[1mEvaluating the workflow at step 15 ...\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:   0%|          | 0/50 [00:00<?, ?it/s]\u001b[A\n",
      "Evaluating workflow:   2%|▏         | 1/50 [00:01<01:34,  1.92s/it]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:   6%|▌         | 3/50 [00:02<00:32,  1.45it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:   8%|▊         | 4/50 [00:02<00:24,  1.84it/s]\u001b[A\n",
      "Evaluating workflow:  14%|█▍        | 7/50 [00:02<00:10,  4.15it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  18%|█▊        | 9/50 [00:03<00:08,  4.61it/s]\u001b[A\n",
      "Evaluating workflow:  22%|██▏       | 11/50 [00:03<00:06,  6.16it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  28%|██▊       | 14/50 [00:03<00:04,  8.56it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  32%|███▏      | 16/50 [00:04<00:05,  5.98it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  40%|████      | 20/50 [00:04<00:03,  8.12it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  44%|████▍     | 22/50 [00:05<00:05,  5.16it/s]\u001b[A\n",
      "Evaluating workflow:  48%|████▊     | 24/50 [00:05<00:04,  6.07it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  52%|█████▏    | 26/50 [00:06<00:06,  3.54it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  62%|██████▏   | 31/50 [00:06<00:03,  5.47it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  66%|██████▌   | 33/50 [00:07<00:02,  5.88it/s]\u001b[A\n",
      "Evaluating workflow:  70%|███████   | 35/50 [00:07<00:02,  7.07it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  78%|███████▊  | 39/50 [00:07<00:01, 10.23it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  82%|████████▏ | 41/50 [00:07<00:01,  8.57it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  86%|████████▌ | 43/50 [00:08<00:01,  6.04it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  90%|█████████ | 45/50 [00:08<00:00,  5.53it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  92%|█████████▏| 46/50 [00:09<00:00,  4.09it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  94%|█████████▍| 47/50 [00:09<00:00,  3.90it/s]\u001b[A\n",
      "Evaluating workflow:  96%|█████████▌| 48/50 [00:09<00:00,  4.14it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  98%|█████████▊| 49/50 [00:10<00:00,  4.76it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow: 100%|██████████| 50/50 [00:10<00:00,  4.87it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "\u001b[32m2026-01-04 09:45:26.729\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m275\u001b[0m - \u001b[1mStep 15 metrics: {'f1': 0.0, 'em': 0.0, 'acc': 0.52}\u001b[0m\n",
      "\u001b[32m2026-01-04 09:45:26.729\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m291\u001b[0m - \u001b[1mMetrics are worse than the best snapshot which has {'f1': 0.0, 'em': 0.0, 'acc': 0.78}. Rolling back to the best snapshot.\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "\r",
      " 75%|███████▌  | 15/20 [32:55<10:34, 126.94s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2026-01-04 09:45:26.732\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36mstep\u001b[0m:\u001b[36m319\u001b[0m - \u001b[1mExecuting workflow...\u001b[0m\n",
      "\u001b[32m2026-01-04 09:45:45.855\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36mstep\u001b[0m:\u001b[36m347\u001b[0m - \u001b[1mComputing gradients...\u001b[0m\n",
      "\u001b[32m2026-01-04 09:47:28.065\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36mstep\u001b[0m:\u001b[36m349\u001b[0m - \u001b[1mUpdating agents...\u001b[0m\n",
      "\u001b[32m2026-01-04 09:47:33.310\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36mstep\u001b[0m:\u001b[36m353\u001b[0m - \u001b[1mAgents updated\u001b[0m\n",
      "\u001b[32m2026-01-04 09:47:33.310\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m271\u001b[0m - \u001b[1mEvaluating the workflow at step 16 ...\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:   0%|          | 0/50 [00:00<?, ?it/s]\u001b[A\n",
      "Evaluating workflow:   2%|▏         | 1/50 [00:01<01:20,  1.64s/it]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  12%|█▏        | 6/50 [00:02<00:14,  3.02it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  16%|█▌        | 8/50 [00:02<00:12,  3.39it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  22%|██▏       | 11/50 [00:03<00:08,  4.76it/s]\u001b[A\n",
      "Evaluating workflow:  32%|███▏      | 16/50 [00:03<00:03,  8.58it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  38%|███▊      | 19/50 [00:03<00:04,  7.63it/s]\u001b[A\n",
      "Evaluating workflow:  44%|████▍     | 22/50 [00:03<00:02,  9.52it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  48%|████▊     | 24/50 [00:04<00:04,  5.57it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  52%|█████▏    | 26/50 [00:04<00:03,  6.19it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  56%|█████▌    | 28/50 [00:05<00:04,  4.65it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  58%|█████▊    | 29/50 [00:05<00:04,  4.49it/s]\u001b[A\n",
      "Evaluating workflow:  64%|██████▍   | 32/50 [00:06<00:02,  6.55it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  68%|██████▊   | 34/50 [00:06<00:02,  5.78it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  72%|███████▏  | 36/50 [00:06<00:02,  6.99it/s]\u001b[A\n",
      "Evaluating workflow:  78%|███████▊  | 39/50 [00:06<00:01,  8.90it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  82%|████████▏ | 41/50 [00:07<00:00,  9.58it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  86%|████████▌ | 43/50 [00:07<00:00,  8.28it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  90%|█████████ | 45/50 [00:07<00:00,  8.79it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  94%|█████████▍| 47/50 [00:07<00:00,  7.07it/s]\u001b[A\n",
      "Evaluating workflow:  96%|█████████▌| 48/50 [00:08<00:00,  6.66it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow: 100%|██████████| 50/50 [00:10<00:00,  4.95it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "\u001b[32m2026-01-04 09:47:43.533\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m275\u001b[0m - \u001b[1mStep 16 metrics: {'f1': 0.0, 'em': 0.0, 'acc': 0.74}\u001b[0m\n",
      "\u001b[32m2026-01-04 09:47:43.534\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m291\u001b[0m - \u001b[1mMetrics are worse than the best snapshot which has {'f1': 0.0, 'em': 0.0, 'acc': 0.78}. Rolling back to the best snapshot.\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "\r",
      " 80%|████████  | 16/20 [35:11<08:39, 129.91s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2026-01-04 09:47:43.537\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36mstep\u001b[0m:\u001b[36m319\u001b[0m - \u001b[1mExecuting workflow...\u001b[0m\n",
      "\u001b[32m2026-01-04 09:47:53.316\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36mstep\u001b[0m:\u001b[36m347\u001b[0m - \u001b[1mComputing gradients...\u001b[0m\n",
      "\u001b[32m2026-01-04 09:48:57.465\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36mstep\u001b[0m:\u001b[36m349\u001b[0m - \u001b[1mUpdating agents...\u001b[0m\n",
      "\u001b[32m2026-01-04 09:49:02.470\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36mstep\u001b[0m:\u001b[36m353\u001b[0m - \u001b[1mAgents updated\u001b[0m\n",
      "\u001b[32m2026-01-04 09:49:02.470\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m271\u001b[0m - \u001b[1mEvaluating the workflow at step 17 ...\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:   0%|          | 0/50 [00:00<?, ?it/s]\u001b[A\n",
      "Evaluating workflow:   2%|▏         | 1/50 [00:01<01:28,  1.80s/it]\u001b[A\n",
      "Evaluating workflow:   8%|▊         | 4/50 [00:01<00:17,  2.60it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  12%|█▏        | 6/50 [00:02<00:11,  3.72it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  14%|█▍        | 7/50 [00:02<00:10,  4.01it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  18%|█▊        | 9/50 [00:03<00:11,  3.65it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  22%|██▏       | 11/50 [00:03<00:08,  4.51it/s]\u001b[A\n",
      "Evaluating workflow:  24%|██▍       | 12/50 [00:03<00:07,  5.08it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  28%|██▊       | 14/50 [00:03<00:05,  6.09it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  30%|███       | 15/50 [00:03<00:07,  4.93it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  38%|███▊      | 19/50 [00:04<00:03,  8.26it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  42%|████▏     | 21/50 [00:04<00:03,  8.55it/s]\u001b[A\n",
      "Evaluating workflow:  48%|████▊     | 24/50 [00:04<00:02, 11.14it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  54%|█████▍    | 27/50 [00:04<00:01, 11.89it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  58%|█████▊    | 29/50 [00:05<00:03,  5.38it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  62%|██████▏   | 31/50 [00:06<00:03,  4.97it/s]\u001b[A\n",
      "Evaluating workflow:  68%|██████▊   | 34/50 [00:06<00:02,  6.94it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  72%|███████▏  | 36/50 [00:06<00:01,  7.09it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  76%|███████▌  | 38/50 [00:07<00:02,  5.95it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  80%|████████  | 40/50 [00:07<00:01,  5.41it/s]\u001b[A\n",
      "Evaluating workflow:  82%|████████▏ | 41/50 [00:07<00:01,  5.60it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  88%|████████▊ | 44/50 [00:07<00:00,  7.40it/s]\u001b[A\n",
      "Evaluating workflow:  92%|█████████▏| 46/50 [00:08<00:00,  8.58it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  96%|█████████▌| 48/50 [00:08<00:00,  8.14it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow: 100%|██████████| 50/50 [00:08<00:00,  5.73it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "\u001b[32m2026-01-04 09:49:11.342\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m275\u001b[0m - \u001b[1mStep 17 metrics: {'f1': 0.0, 'em': 0.0, 'acc': 0.68}\u001b[0m\n",
      "\u001b[32m2026-01-04 09:49:11.343\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m291\u001b[0m - \u001b[1mMetrics are worse than the best snapshot which has {'f1': 0.0, 'em': 0.0, 'acc': 0.78}. Rolling back to the best snapshot.\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "\r",
      " 85%|████████▌ | 17/20 [36:39<05:51, 117.25s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2026-01-04 09:49:11.346\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36mstep\u001b[0m:\u001b[36m319\u001b[0m - \u001b[1mExecuting workflow...\u001b[0m\n",
      "\u001b[32m2026-01-04 09:49:26.925\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36mstep\u001b[0m:\u001b[36m347\u001b[0m - \u001b[1mComputing gradients...\u001b[0m\n",
      "\u001b[32m2026-01-04 09:51:08.354\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36mstep\u001b[0m:\u001b[36m349\u001b[0m - \u001b[1mUpdating agents...\u001b[0m\n",
      "\u001b[32m2026-01-04 09:51:13.968\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36mstep\u001b[0m:\u001b[36m353\u001b[0m - \u001b[1mAgents updated\u001b[0m\n",
      "\u001b[32m2026-01-04 09:51:13.968\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m271\u001b[0m - \u001b[1mEvaluating the workflow at step 18 ...\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:   0%|          | 0/50 [00:00<?, ?it/s]\u001b[A\n",
      "Evaluating workflow:   2%|▏         | 1/50 [00:01<01:21,  1.66s/it]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:   8%|▊         | 4/50 [00:01<00:17,  2.58it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  10%|█         | 5/50 [00:02<00:16,  2.80it/s]\u001b[A\n",
      "Evaluating workflow:  14%|█▍        | 7/50 [00:02<00:10,  4.18it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  16%|█▌        | 8/50 [00:02<00:09,  4.39it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  18%|█▊        | 9/50 [00:03<00:12,  3.29it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  20%|██        | 10/50 [00:03<00:13,  3.04it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  24%|██▍       | 12/50 [00:03<00:10,  3.64it/s]\u001b[A\n",
      "Evaluating workflow:  26%|██▌       | 13/50 [00:04<00:08,  4.28it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  32%|███▏      | 16/50 [00:04<00:05,  6.39it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  46%|████▌     | 23/50 [00:04<00:02, 12.59it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  52%|█████▏    | 26/50 [00:04<00:01, 12.87it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  56%|█████▌    | 28/50 [00:05<00:02,  9.96it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  60%|██████    | 30/50 [00:06<00:04,  4.71it/s]\u001b[A\n",
      "Evaluating workflow:  64%|██████▍   | 32/50 [00:06<00:03,  5.79it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  68%|██████▊   | 34/50 [00:06<00:02,  6.74it/s]\u001b[A\n",
      "Evaluating workflow:  72%|███████▏  | 36/50 [00:06<00:01,  7.73it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  76%|███████▌  | 38/50 [00:06<00:01,  7.30it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  80%|████████  | 40/50 [00:07<00:01,  7.34it/s]\u001b[A\n",
      "Evaluating workflow:  82%|████████▏ | 41/50 [00:07<00:01,  6.86it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  84%|████████▍ | 42/50 [00:07<00:01,  6.65it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  86%|████████▌ | 43/50 [00:07<00:01,  6.07it/s]\u001b[A\n",
      "Evaluating workflow:  90%|█████████ | 45/50 [00:07<00:00,  7.86it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  92%|█████████▏| 46/50 [00:08<00:00,  4.69it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  96%|█████████▌| 48/50 [00:08<00:00,  5.33it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  98%|█████████▊| 49/50 [00:09<00:00,  3.55it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow: 100%|██████████| 50/50 [00:10<00:00,  4.74it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "\u001b[32m2026-01-04 09:51:24.647\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m275\u001b[0m - \u001b[1mStep 18 metrics: {'f1': 0.0, 'em': 0.0, 'acc': 0.66}\u001b[0m\n",
      "\u001b[32m2026-01-04 09:51:24.647\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m291\u001b[0m - \u001b[1mMetrics are worse than the best snapshot which has {'f1': 0.0, 'em': 0.0, 'acc': 0.78}. Rolling back to the best snapshot.\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "\r",
      " 90%|█████████ | 18/20 [38:52<04:04, 122.07s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2026-01-04 09:51:24.651\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36mstep\u001b[0m:\u001b[36m319\u001b[0m - \u001b[1mExecuting workflow...\u001b[0m\n",
      "\u001b[32m2026-01-04 09:51:43.880\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36mstep\u001b[0m:\u001b[36m347\u001b[0m - \u001b[1mComputing gradients...\u001b[0m\n",
      "\u001b[32m2026-01-04 09:53:23.772\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36mstep\u001b[0m:\u001b[36m349\u001b[0m - \u001b[1mUpdating agents...\u001b[0m\n",
      "\u001b[32m2026-01-04 09:53:29.558\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36mstep\u001b[0m:\u001b[36m353\u001b[0m - \u001b[1mAgents updated\u001b[0m\n",
      "\u001b[32m2026-01-04 09:53:29.559\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m271\u001b[0m - \u001b[1mEvaluating the workflow at step 19 ...\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:   0%|          | 0/50 [00:00<?, ?it/s]\u001b[A\n",
      "Evaluating workflow:   2%|▏         | 1/50 [00:01<01:15,  1.54s/it]\u001b[A\n",
      "Evaluating workflow:  10%|█         | 5/50 [00:01<00:11,  3.98it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  18%|█▊        | 9/50 [00:02<00:07,  5.26it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  22%|██▏       | 11/50 [00:03<00:10,  3.64it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  26%|██▌       | 13/50 [00:03<00:08,  4.25it/s]\u001b[A\n",
      "Evaluating workflow:  36%|███▌      | 18/50 [00:03<00:04,  7.90it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  42%|████▏     | 21/50 [00:03<00:02, 10.06it/s]\u001b[A\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  48%|████▊     | 24/50 [00:03<00:02, 11.17it/s]\u001b[A\n",
      "Evaluating workflow:  54%|█████▍    | 27/50 [00:04<00:01, 12.75it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  60%|██████    | 30/50 [00:05<00:03,  5.85it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  64%|██████▍   | 32/50 [00:05<00:03,  5.23it/s]\u001b[A\n",
      "Evaluating workflow:  68%|██████▊   | 34/50 [00:05<00:02,  5.99it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  72%|███████▏  | 36/50 [00:06<00:02,  6.43it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  76%|███████▌  | 38/50 [00:06<00:01,  6.83it/s]\u001b[A\n",
      "Evaluating workflow:  80%|████████  | 40/50 [00:06<00:01,  8.03it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  84%|████████▍ | 42/50 [00:06<00:01,  6.64it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  88%|████████▊ | 44/50 [00:07<00:00,  6.72it/s]\u001b[A\n",
      "Evaluating workflow:  92%|█████████▏| 46/50 [00:07<00:00,  7.61it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  96%|█████████▌| 48/50 [00:07<00:00,  5.80it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  98%|█████████▊| 49/50 [00:08<00:00,  4.72it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow: 100%|██████████| 50/50 [00:08<00:00,  5.82it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "\u001b[32m2026-01-04 09:53:38.265\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m275\u001b[0m - \u001b[1mStep 19 metrics: {'f1': 0.0, 'em': 0.0, 'acc': 0.76}\u001b[0m\n",
      "\u001b[32m2026-01-04 09:53:38.267\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m291\u001b[0m - \u001b[1mMetrics are worse than the best snapshot which has {'f1': 0.0, 'em': 0.0, 'acc': 0.78}. Rolling back to the best snapshot.\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "\r",
      " 95%|█████████▌| 19/20 [41:06<02:05, 125.54s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2026-01-04 09:53:38.270\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36mstep\u001b[0m:\u001b[36m319\u001b[0m - \u001b[1mExecuting workflow...\u001b[0m\n",
      "\u001b[32m2026-01-04 09:53:57.435\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36mstep\u001b[0m:\u001b[36m347\u001b[0m - \u001b[1mComputing gradients...\u001b[0m\n",
      "\u001b[32m2026-01-04 09:55:46.089\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36mstep\u001b[0m:\u001b[36m349\u001b[0m - \u001b[1mUpdating agents...\u001b[0m\n",
      "\u001b[32m2026-01-04 09:55:51.861\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36mstep\u001b[0m:\u001b[36m353\u001b[0m - \u001b[1mAgents updated\u001b[0m\n",
      "\u001b[32m2026-01-04 09:55:51.861\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m271\u001b[0m - \u001b[1mEvaluating the workflow at step 20 ...\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:   0%|          | 0/50 [00:00<?, ?it/s]\u001b[A\n",
      "Evaluating workflow:   2%|▏         | 1/50 [00:01<01:07,  1.38s/it]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:   4%|▍         | 2/50 [00:01<00:33,  1.44it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:   6%|▌         | 3/50 [00:01<00:22,  2.06it/s]\u001b[A\n",
      "Evaluating workflow:   8%|▊         | 4/50 [00:02<00:16,  2.76it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  12%|█▏        | 6/50 [00:02<00:08,  5.03it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  18%|█▊        | 9/50 [00:02<00:05,  7.24it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  24%|██▍       | 12/50 [00:03<00:07,  4.88it/s]\u001b[A\n",
      "Evaluating workflow:  28%|██▊       | 14/50 [00:03<00:05,  6.24it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  32%|███▏      | 16/50 [00:03<00:05,  6.48it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  36%|███▌      | 18/50 [00:03<00:04,  6.76it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  42%|████▏     | 21/50 [00:04<00:03,  7.76it/s]\u001b[A\n",
      "Evaluating workflow:  52%|█████▏    | 26/50 [00:04<00:01, 12.14it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  56%|█████▌    | 28/50 [00:04<00:01, 13.15it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  60%|██████    | 30/50 [00:05<00:02,  7.65it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  64%|██████▍   | 32/50 [00:05<00:02,  7.32it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  68%|██████▊   | 34/50 [00:05<00:02,  5.75it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  70%|███████   | 35/50 [00:06<00:02,  5.52it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  76%|███████▌  | 38/50 [00:06<00:01,  6.56it/s]\u001b[A\n",
      "Evaluating workflow:  78%|███████▊  | 39/50 [00:06<00:01,  6.47it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  84%|████████▍ | 42/50 [00:06<00:00,  9.39it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  88%|████████▊ | 44/50 [00:07<00:00,  6.45it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:  96%|█████████▌| 48/50 [00:07<00:00,  8.75it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow: 100%|██████████| 50/50 [00:08<00:00,  6.09it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "\u001b[32m2026-01-04 09:56:00.199\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m275\u001b[0m - \u001b[1mStep 20 metrics: {'f1': 0.0, 'em': 0.0, 'acc': 0.72}\u001b[0m\n",
      "\u001b[32m2026-01-04 09:56:00.199\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m291\u001b[0m - \u001b[1mMetrics are worse than the best snapshot which has {'f1': 0.0, 'em': 0.0, 'acc': 0.78}. Rolling back to the best snapshot.\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "100%|██████████| 20/20 [43:28<00:00, 130.42s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2026-01-04 09:56:00.203\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m300\u001b[0m - \u001b[1mReached the maximum number of steps 20. Optimization has finished.\u001b[0m\n",
      "\u001b[32m2026-01-04 09:56:00.203\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.workflow.workflow_graph\u001b[0m:\u001b[36msave_module\u001b[0m:\u001b[36m1204\u001b[0m - \u001b[1mSaving SequentialWorkFlowGraph to ./PertQA_textgrad_final.json\u001b[0m\n",
      "\u001b[32m2026-01-04 09:56:00.213\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.workflow.workflow_graph\u001b[0m:\u001b[36msave_module\u001b[0m:\u001b[36m1204\u001b[0m - \u001b[1mSaving SequentialWorkFlowGraph to ./PertQA_textgrad_best.json\u001b[0m\n",
      "\u001b[32m2026-01-04 09:56:00.214\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.textgrad_optimizer\u001b[0m:\u001b[36mrestore_best_graph\u001b[0m:\u001b[36m448\u001b[0m - \u001b[1mRestored the best graph from snapshot with metrics {'f1': 0.0, 'em': 0.0, 'acc': 0.78}\u001b[0m\n",
      "\u001b[32m2026-01-04 09:56:00.215\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36m__main__\u001b[0m:\u001b[36m<module>\u001b[0m:\u001b[36m10\u001b[0m - \u001b[1mEvaluating workflow on test set...\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluating workflow:   0%|          | 3/3000 [00:01<20:53,  2.39it/s]  "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   0%|          | 5/3000 [00:01<13:20,  3.74it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   0%|          | 7/3000 [00:02<10:18,  4.84it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   0%|          | 9/3000 [00:02<08:20,  5.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   0%|          | 11/3000 [00:02<10:27,  4.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   0%|          | 13/3000 [00:03<09:47,  5.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   0%|          | 15/3000 [00:03<12:07,  4.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   1%|          | 20/3000 [00:04<05:16,  9.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   1%|          | 22/3000 [00:04<05:41,  8.72it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   1%|          | 28/3000 [00:04<03:59, 12.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   1%|          | 30/3000 [00:04<03:47, 13.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   1%|          | 35/3000 [00:05<06:23,  7.74it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   1%|          | 37/3000 [00:06<06:18,  7.82it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   1%|▏         | 41/3000 [00:07<07:56,  6.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   1%|▏         | 42/3000 [00:07<10:02,  4.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   2%|▏         | 46/3000 [00:07<06:59,  7.05it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   2%|▏         | 49/3000 [00:08<08:01,  6.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   2%|▏         | 50/3000 [00:08<09:06,  5.40it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   2%|▏         | 54/3000 [00:09<06:49,  7.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   2%|▏         | 59/3000 [00:09<04:53, 10.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   2%|▏         | 61/3000 [00:10<06:38,  7.37it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   2%|▏         | 62/3000 [00:10<07:33,  6.47it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   2%|▏         | 66/3000 [00:11<07:47,  6.28it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   2%|▏         | 68/3000 [00:11<07:32,  6.48it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   2%|▏         | 70/3000 [00:11<06:02,  8.09it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   2%|▏         | 74/3000 [00:11<05:09,  9.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   3%|▎         | 76/3000 [00:12<06:44,  7.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   3%|▎         | 77/3000 [00:12<09:35,  5.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   3%|▎         | 78/3000 [00:12<09:48,  4.97it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   3%|▎         | 81/3000 [00:13<07:41,  6.32it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   3%|▎         | 84/3000 [00:13<06:11,  7.84it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   3%|▎         | 86/3000 [00:13<04:53,  9.93it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   3%|▎         | 92/3000 [00:14<04:06, 11.82it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   3%|▎         | 94/3000 [00:14<07:18,  6.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   3%|▎         | 96/3000 [00:15<08:35,  5.63it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   3%|▎         | 99/3000 [00:15<06:47,  7.11it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   3%|▎         | 103/3000 [00:15<03:54, 12.37it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   4%|▎         | 105/3000 [00:16<06:42,  7.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   4%|▎         | 110/3000 [00:16<05:53,  8.17it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   4%|▍         | 116/3000 [00:17<05:03,  9.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   4%|▍         | 118/3000 [00:17<04:54,  9.80it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   4%|▍         | 121/3000 [00:18<05:59,  8.02it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   4%|▍         | 123/3000 [00:18<07:48,  6.14it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   4%|▍         | 126/3000 [00:19<07:39,  6.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   4%|▍         | 128/3000 [00:19<07:20,  6.52it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   5%|▍         | 137/3000 [00:20<05:04,  9.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   5%|▍         | 139/3000 [00:20<05:49,  8.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   5%|▍         | 143/3000 [00:21<07:48,  6.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   5%|▍         | 148/3000 [00:21<05:17,  8.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   5%|▌         | 150/3000 [00:22<06:29,  7.32it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   5%|▌         | 152/3000 [00:22<06:33,  7.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   5%|▌         | 153/3000 [00:22<07:02,  6.74it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   5%|▌         | 157/3000 [00:23<05:51,  8.09it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   5%|▌         | 161/3000 [00:23<05:51,  8.07it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   5%|▌         | 164/3000 [00:24<06:28,  7.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   6%|▌         | 166/3000 [00:24<05:25,  8.72it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   6%|▌         | 168/3000 [00:24<06:39,  7.09it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   6%|▌         | 171/3000 [00:25<06:32,  7.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   6%|▌         | 174/3000 [00:25<04:35, 10.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   6%|▌         | 176/3000 [00:25<05:06,  9.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   6%|▌         | 178/3000 [00:26<06:25,  7.32it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   6%|▌         | 179/3000 [00:26<06:39,  7.06it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   6%|▌         | 182/3000 [00:26<07:02,  6.67it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   6%|▌         | 184/3000 [00:27<07:13,  6.49it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   6%|▌         | 187/3000 [00:27<05:16,  8.88it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   6%|▋         | 188/3000 [00:27<06:55,  6.77it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   6%|▋         | 191/3000 [00:28<06:45,  6.93it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   6%|▋         | 192/3000 [00:28<06:43,  6.96it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   6%|▋         | 195/3000 [00:28<05:56,  7.88it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   7%|▋         | 198/3000 [00:28<05:59,  7.80it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   7%|▋         | 201/3000 [00:29<05:36,  8.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   7%|▋         | 202/3000 [00:29<05:26,  8.57it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   7%|▋         | 206/3000 [00:29<05:10,  9.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   7%|▋         | 210/3000 [00:30<05:01,  9.27it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   7%|▋         | 212/3000 [00:30<07:09,  6.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   7%|▋         | 213/3000 [00:31<07:21,  6.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   7%|▋         | 216/3000 [00:31<06:10,  7.52it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   7%|▋         | 219/3000 [00:31<06:13,  7.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   7%|▋         | 221/3000 [00:31<05:47,  7.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   7%|▋         | 224/3000 [00:32<03:37, 12.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   8%|▊         | 226/3000 [00:32<05:16,  8.77it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   8%|▊         | 228/3000 [00:32<07:30,  6.16it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   8%|▊         | 230/3000 [00:33<08:03,  5.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   8%|▊         | 235/3000 [00:33<05:22,  8.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   8%|▊         | 239/3000 [00:34<06:34,  7.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   8%|▊         | 241/3000 [00:34<07:24,  6.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   8%|▊         | 243/3000 [00:35<07:35,  6.05it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   8%|▊         | 245/3000 [00:35<07:16,  6.32it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   8%|▊         | 249/3000 [00:35<04:58,  9.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   8%|▊         | 251/3000 [00:36<05:28,  8.37it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   8%|▊         | 254/3000 [00:36<07:07,  6.42it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   8%|▊         | 255/3000 [00:36<06:41,  6.83it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   9%|▊         | 259/3000 [00:37<05:43,  7.97it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   9%|▉         | 263/3000 [00:38<05:51,  7.78it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   9%|▉         | 265/3000 [00:38<05:59,  7.61it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   9%|▉         | 269/3000 [00:39<08:40,  5.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   9%|▉         | 271/3000 [00:39<06:55,  6.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   9%|▉         | 275/3000 [00:40<05:51,  7.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   9%|▉         | 277/3000 [00:40<05:39,  8.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   9%|▉         | 279/3000 [00:40<05:16,  8.61it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   9%|▉         | 282/3000 [00:40<03:58, 11.37it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   9%|▉         | 284/3000 [00:41<06:21,  7.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  10%|▉         | 286/3000 [00:41<06:44,  6.71it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  10%|▉         | 289/3000 [00:41<06:39,  6.78it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  10%|▉         | 292/3000 [00:42<06:39,  6.78it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  10%|▉         | 295/3000 [00:43<09:21,  4.82it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  10%|█         | 300/3000 [00:43<05:44,  7.84it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  10%|█         | 306/3000 [00:43<03:45, 11.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  10%|█         | 308/3000 [00:44<04:58,  9.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  10%|█         | 310/3000 [00:44<07:09,  6.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  10%|█         | 315/3000 [00:45<05:55,  7.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  11%|█         | 318/3000 [00:45<07:00,  6.37it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  11%|█         | 320/3000 [00:46<06:33,  6.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  11%|█         | 321/3000 [00:46<06:59,  6.38it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  11%|█         | 324/3000 [00:46<06:12,  7.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  11%|█         | 328/3000 [00:47<03:57, 11.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  11%|█         | 332/3000 [00:47<03:23, 13.11it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  11%|█         | 334/3000 [00:47<03:44, 11.88it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  11%|█▏        | 339/3000 [00:47<03:54, 11.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  11%|█▏        | 341/3000 [00:48<07:44,  5.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  11%|█▏        | 343/3000 [00:49<08:18,  5.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  11%|█▏        | 344/3000 [00:49<07:46,  5.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  12%|█▏        | 346/3000 [00:49<06:47,  6.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  12%|█▏        | 351/3000 [00:49<04:47,  9.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  12%|█▏        | 353/3000 [00:50<06:18,  7.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  12%|█▏        | 357/3000 [00:50<05:20,  8.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  12%|█▏        | 359/3000 [00:51<06:37,  6.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  12%|█▏        | 364/3000 [00:51<04:03, 10.84it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  12%|█▏        | 367/3000 [00:51<04:04, 10.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  12%|█▏        | 371/3000 [00:52<03:56, 11.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  12%|█▏        | 373/3000 [00:53<07:38,  5.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  12%|█▎        | 375/3000 [00:53<06:50,  6.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  13%|█▎        | 377/3000 [00:53<07:20,  5.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  13%|█▎        | 381/3000 [00:54<05:54,  7.38it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  13%|█▎        | 383/3000 [00:54<04:59,  8.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  13%|█▎        | 385/3000 [00:54<05:56,  7.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  13%|█▎        | 387/3000 [00:54<05:07,  8.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  13%|█▎        | 389/3000 [00:54<04:53,  8.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  13%|█▎        | 394/3000 [00:55<04:59,  8.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  13%|█▎        | 396/3000 [00:55<04:39,  9.32it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  13%|█▎        | 401/3000 [00:56<05:13,  8.29it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  13%|█▎        | 403/3000 [00:57<07:06,  6.09it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  13%|█▎        | 404/3000 [00:57<08:01,  5.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  14%|█▎        | 408/3000 [00:57<05:57,  7.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  14%|█▎        | 410/3000 [00:58<06:58,  6.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  14%|█▍        | 416/3000 [00:58<03:48, 11.29it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  14%|█▍        | 418/3000 [00:59<05:03,  8.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  14%|█▍        | 420/3000 [00:59<05:27,  7.89it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  14%|█▍        | 422/3000 [00:59<05:37,  7.63it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  14%|█▍        | 424/3000 [01:00<06:41,  6.42it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  14%|█▍        | 427/3000 [01:00<05:51,  7.32it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  14%|█▍        | 431/3000 [01:00<05:42,  7.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  14%|█▍        | 433/3000 [01:01<06:24,  6.67it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  14%|█▍        | 434/3000 [01:01<07:16,  5.87it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  15%|█▍        | 437/3000 [01:01<05:51,  7.29it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  15%|█▍        | 440/3000 [01:02<05:41,  7.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  15%|█▍        | 442/3000 [01:02<05:31,  7.72it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  15%|█▍        | 444/3000 [01:02<06:03,  7.04it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  15%|█▍        | 446/3000 [01:03<06:07,  6.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  15%|█▍        | 448/3000 [01:03<06:04,  7.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  15%|█▍        | 449/3000 [01:03<06:45,  6.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  15%|█▌        | 453/3000 [01:04<04:50,  8.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  15%|█▌        | 454/3000 [01:04<05:09,  8.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  15%|█▌        | 455/3000 [01:04<06:50,  6.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  15%|█▌        | 458/3000 [01:04<06:08,  6.89it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  15%|█▌        | 461/3000 [01:05<05:02,  8.40it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  15%|█▌        | 463/3000 [01:05<04:01, 10.48it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  16%|█▌        | 465/3000 [01:05<04:34,  9.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  16%|█▌        | 469/3000 [01:06<05:18,  7.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  16%|█▌        | 470/3000 [01:06<05:34,  7.57it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  16%|█▌        | 472/3000 [01:06<05:56,  7.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  16%|█▌        | 474/3000 [01:07<06:43,  6.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  16%|█▌        | 476/3000 [01:07<07:11,  5.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  16%|█▌        | 481/3000 [01:07<03:43, 11.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  16%|█▌        | 483/3000 [01:08<04:45,  8.83it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  16%|█▌        | 487/3000 [01:08<05:44,  7.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  16%|█▋        | 489/3000 [01:08<04:44,  8.84it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  16%|█▋        | 491/3000 [01:09<05:03,  8.28it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  16%|█▋        | 493/3000 [01:09<04:49,  8.67it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  17%|█▋        | 496/3000 [01:09<04:25,  9.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  17%|█▋        | 498/3000 [01:10<07:26,  5.60it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  17%|█▋        | 501/3000 [01:10<05:51,  7.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  17%|█▋        | 503/3000 [01:10<05:16,  7.89it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  17%|█▋        | 508/3000 [01:11<04:15,  9.77it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  17%|█▋        | 510/3000 [01:11<04:41,  8.83it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  17%|█▋        | 514/3000 [01:12<05:53,  7.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  17%|█▋        | 515/3000 [01:12<06:47,  6.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  17%|█▋        | 518/3000 [01:13<07:27,  5.55it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  17%|█▋        | 520/3000 [01:13<05:39,  7.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  17%|█▋        | 522/3000 [01:13<05:59,  6.89it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  17%|█▋        | 524/3000 [01:14<05:37,  7.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  18%|█▊        | 526/3000 [01:14<04:56,  8.35it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  18%|█▊        | 529/3000 [01:14<06:29,  6.35it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  18%|█▊        | 530/3000 [01:15<06:40,  6.17it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  18%|█▊        | 534/3000 [01:15<05:25,  7.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  18%|█▊        | 536/3000 [01:15<06:10,  6.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  18%|█▊        | 539/3000 [01:16<05:06,  8.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  18%|█▊        | 541/3000 [01:16<05:22,  7.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  18%|█▊        | 543/3000 [01:16<05:04,  8.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  18%|█▊        | 546/3000 [01:17<04:55,  8.32it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  18%|█▊        | 548/3000 [01:17<05:01,  8.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  18%|█▊        | 550/3000 [01:17<05:09,  7.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  18%|█▊        | 553/3000 [01:17<05:10,  7.87it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  18%|█▊        | 554/3000 [01:18<05:05,  8.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  19%|█▊        | 556/3000 [01:18<07:30,  5.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  19%|█▊        | 559/3000 [01:19<07:01,  5.80it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  19%|█▊        | 562/3000 [01:19<04:48,  8.46it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  19%|█▉        | 565/3000 [01:19<04:56,  8.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  19%|█▉        | 566/3000 [01:20<06:46,  5.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  19%|█▉        | 572/3000 [01:20<03:34, 11.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  19%|█▉        | 574/3000 [01:20<04:29,  9.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  19%|█▉        | 576/3000 [01:21<04:43,  8.55it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  19%|█▉        | 578/3000 [01:21<07:56,  5.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  19%|█▉        | 581/3000 [01:22<06:22,  6.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  19%|█▉        | 582/3000 [01:22<07:20,  5.49it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  20%|█▉        | 586/3000 [01:23<05:54,  6.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  20%|█▉        | 588/3000 [01:23<05:17,  7.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  20%|█▉        | 589/3000 [01:23<09:03,  4.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  20%|█▉        | 592/3000 [01:24<07:41,  5.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  20%|█▉        | 598/3000 [01:24<04:44,  8.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  20%|██        | 600/3000 [01:24<04:18,  9.28it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  20%|██        | 602/3000 [01:25<05:58,  6.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  20%|██        | 604/3000 [01:25<05:02,  7.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  20%|██        | 606/3000 [01:26<06:16,  6.36it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  20%|██        | 607/3000 [01:26<06:35,  6.05it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  20%|██        | 609/3000 [01:26<07:16,  5.47it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  20%|██        | 610/3000 [01:26<06:59,  5.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  20%|██        | 611/3000 [01:27<09:11,  4.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  20%|██        | 613/3000 [01:27<07:33,  5.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  21%|██        | 617/3000 [01:28<07:01,  5.66it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  21%|██        | 620/3000 [01:28<05:09,  7.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  21%|██        | 623/3000 [01:28<05:03,  7.83it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  21%|██        | 625/3000 [01:29<04:40,  8.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  21%|██        | 628/3000 [01:29<05:11,  7.60it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  21%|██        | 630/3000 [01:30<06:49,  5.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  21%|██        | 634/3000 [01:30<06:58,  5.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  21%|██        | 636/3000 [01:30<05:33,  7.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  21%|██▏       | 641/3000 [01:31<03:48, 10.32it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  21%|██▏       | 643/3000 [01:31<03:28, 11.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  22%|██▏       | 645/3000 [01:31<04:46,  8.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  22%|██▏       | 647/3000 [01:32<05:57,  6.57it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  22%|██▏       | 649/3000 [01:32<06:18,  6.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  22%|██▏       | 650/3000 [01:32<06:37,  5.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  22%|██▏       | 652/3000 [01:33<07:16,  5.37it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  22%|██▏       | 655/3000 [01:33<05:36,  6.97it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  22%|██▏       | 657/3000 [01:33<05:17,  7.37it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  22%|██▏       | 659/3000 [01:34<07:11,  5.42it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  22%|██▏       | 661/3000 [01:34<05:12,  7.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  22%|██▏       | 664/3000 [01:34<05:07,  7.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  22%|██▏       | 666/3000 [01:35<05:38,  6.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  22%|██▏       | 668/3000 [01:35<06:30,  5.96it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  22%|██▏       | 670/3000 [01:35<06:16,  6.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  22%|██▏       | 672/3000 [01:36<06:58,  5.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  23%|██▎       | 676/3000 [01:36<04:33,  8.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  23%|██▎       | 679/3000 [01:37<05:53,  6.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  23%|██▎       | 681/3000 [01:37<04:31,  8.55it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  23%|██▎       | 684/3000 [01:37<05:15,  7.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  23%|██▎       | 687/3000 [01:38<04:23,  8.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  23%|██▎       | 688/3000 [01:38<06:09,  6.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  23%|██▎       | 692/3000 [01:38<04:29,  8.57it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  23%|██▎       | 695/3000 [01:39<03:42, 10.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  23%|██▎       | 697/3000 [01:39<04:46,  8.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  23%|██▎       | 698/3000 [01:40<07:24,  5.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  23%|██▎       | 700/3000 [01:40<07:12,  5.32it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  23%|██▎       | 703/3000 [01:40<05:51,  6.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  24%|██▎       | 707/3000 [01:41<04:53,  7.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  24%|██▎       | 709/3000 [01:41<04:40,  8.16it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  24%|██▎       | 711/3000 [01:41<06:19,  6.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  24%|██▍       | 713/3000 [01:42<05:41,  6.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  24%|██▍       | 718/3000 [01:42<03:11, 11.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  24%|██▍       | 720/3000 [01:42<04:50,  7.84it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  24%|██▍       | 722/3000 [01:43<07:49,  4.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  24%|██▍       | 724/3000 [01:43<06:48,  5.58it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  24%|██▍       | 727/3000 [01:44<05:22,  7.05it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  24%|██▍       | 729/3000 [01:44<06:47,  5.57it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  24%|██▍       | 732/3000 [01:44<04:42,  8.02it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  24%|██▍       | 734/3000 [01:45<04:53,  7.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  25%|██▍       | 736/3000 [01:45<06:47,  5.55it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  25%|██▍       | 739/3000 [01:46<06:08,  6.14it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  25%|██▍       | 741/3000 [01:46<05:52,  6.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  25%|██▍       | 744/3000 [01:46<04:59,  7.52it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  25%|██▍       | 747/3000 [01:47<04:20,  8.66it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  25%|██▍       | 749/3000 [01:47<04:50,  7.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  25%|██▌       | 750/3000 [01:48<10:48,  3.47it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  25%|██▌       | 754/3000 [01:48<07:08,  5.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  25%|██▌       | 755/3000 [01:49<06:53,  5.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  25%|██▌       | 757/3000 [01:49<06:19,  5.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  25%|██▌       | 759/3000 [01:50<09:01,  4.14it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  26%|██▌       | 765/3000 [01:50<05:55,  6.28it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  26%|██▌       | 768/3000 [01:50<04:36,  8.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  26%|██▌       | 769/3000 [01:51<05:24,  6.88it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  26%|██▌       | 773/3000 [01:51<04:24,  8.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  26%|██▌       | 775/3000 [01:52<06:24,  5.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  26%|██▌       | 776/3000 [01:52<05:49,  6.37it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  26%|██▌       | 779/3000 [01:52<05:07,  7.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  26%|██▌       | 780/3000 [01:52<05:34,  6.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  26%|██▌       | 783/3000 [01:53<04:50,  7.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  26%|██▌       | 784/3000 [01:53<04:47,  7.71it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  26%|██▌       | 786/3000 [01:53<06:10,  5.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  26%|██▌       | 787/3000 [01:53<06:33,  5.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  26%|██▋       | 790/3000 [01:54<05:14,  7.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  26%|██▋       | 795/3000 [01:54<03:29, 10.52it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  27%|██▋       | 797/3000 [01:54<03:46,  9.72it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  27%|██▋       | 800/3000 [01:55<04:47,  7.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  27%|██▋       | 804/3000 [01:55<04:37,  7.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  27%|██▋       | 806/3000 [01:56<07:19,  4.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  27%|██▋       | 807/3000 [01:56<07:34,  4.83it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  27%|██▋       | 811/3000 [01:57<05:33,  6.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  27%|██▋       | 814/3000 [01:57<05:04,  7.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  27%|██▋       | 818/3000 [01:58<04:41,  7.74it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  27%|██▋       | 820/3000 [01:58<03:51,  9.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  27%|██▋       | 824/3000 [01:58<03:46,  9.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  28%|██▊       | 826/3000 [01:59<04:22,  8.27it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  28%|██▊       | 828/3000 [01:59<04:16,  8.47it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  28%|██▊       | 830/3000 [01:59<06:21,  5.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  28%|██▊       | 832/3000 [02:00<05:48,  6.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  28%|██▊       | 834/3000 [02:00<05:00,  7.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  28%|██▊       | 837/3000 [02:00<05:23,  6.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  28%|██▊       | 840/3000 [02:01<05:33,  6.48it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  28%|██▊       | 842/3000 [02:01<05:31,  6.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  28%|██▊       | 845/3000 [02:01<04:19,  8.29it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  28%|██▊       | 847/3000 [02:02<04:11,  8.55it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  28%|██▊       | 852/3000 [02:02<02:39, 13.47it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  28%|██▊       | 854/3000 [02:02<03:25, 10.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  29%|██▊       | 856/3000 [02:03<04:42,  7.60it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  29%|██▊       | 859/3000 [02:03<05:48,  6.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  29%|██▊       | 862/3000 [02:03<04:03,  8.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  29%|██▉       | 864/3000 [02:04<03:40,  9.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  29%|██▉       | 866/3000 [02:04<04:50,  7.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  29%|██▉       | 869/3000 [02:05<05:34,  6.36it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  29%|██▉       | 871/3000 [02:05<04:34,  7.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  29%|██▉       | 872/3000 [02:05<04:40,  7.58it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  29%|██▉       | 873/3000 [02:05<05:33,  6.38it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  29%|██▉       | 877/3000 [02:06<04:59,  7.09it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  29%|██▉       | 881/3000 [02:06<04:14,  8.32it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  29%|██▉       | 883/3000 [02:06<03:58,  8.89it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  30%|██▉       | 888/3000 [02:07<04:03,  8.67it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  30%|██▉       | 890/3000 [02:07<04:39,  7.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  30%|██▉       | 892/3000 [02:07<04:42,  7.47it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  30%|██▉       | 893/3000 [02:08<06:01,  5.83it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  30%|██▉       | 895/3000 [02:08<05:50,  6.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  30%|██▉       | 897/3000 [02:08<06:02,  5.80it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  30%|███       | 902/3000 [02:09<04:10,  8.37it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  30%|███       | 904/3000 [02:09<04:53,  7.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  30%|███       | 906/3000 [02:10<05:02,  6.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  30%|███       | 911/3000 [02:10<03:50,  9.06it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  30%|███       | 912/3000 [02:10<03:49,  9.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  30%|███       | 914/3000 [02:11<05:41,  6.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  31%|███       | 916/3000 [02:11<04:32,  7.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  31%|███       | 917/3000 [02:11<04:44,  7.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  31%|███       | 919/3000 [02:11<05:01,  6.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  31%|███       | 922/3000 [02:12<05:01,  6.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  31%|███       | 926/3000 [02:12<04:17,  8.05it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  31%|███       | 928/3000 [02:13<05:17,  6.52it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  31%|███       | 932/3000 [02:13<04:10,  8.27it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  31%|███       | 934/3000 [02:13<04:52,  7.07it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  31%|███▏      | 938/3000 [02:14<03:55,  8.77it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  31%|███▏      | 941/3000 [02:14<04:27,  7.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  31%|███▏      | 942/3000 [02:14<04:55,  6.97it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  32%|███▏      | 945/3000 [02:15<04:43,  7.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  32%|███▏      | 947/3000 [02:15<04:48,  7.11it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  32%|███▏      | 950/3000 [02:15<04:20,  7.86it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  32%|███▏      | 951/3000 [02:16<05:17,  6.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  32%|███▏      | 957/3000 [02:16<03:30,  9.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  32%|███▏      | 959/3000 [02:17<04:35,  7.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  32%|███▏      | 961/3000 [02:17<04:20,  7.83it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  32%|███▏      | 963/3000 [02:17<04:42,  7.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  32%|███▏      | 968/3000 [02:18<04:05,  8.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  32%|███▏      | 971/3000 [02:18<03:32,  9.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  32%|███▏      | 973/3000 [02:18<04:20,  7.78it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  32%|███▎      | 975/3000 [02:19<05:56,  5.67it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  33%|███▎      | 981/3000 [02:20<03:37,  9.27it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  33%|███▎      | 983/3000 [02:20<05:21,  6.27it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  33%|███▎      | 989/3000 [02:21<03:14, 10.36it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  33%|███▎      | 991/3000 [02:21<04:16,  7.82it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  33%|███▎      | 993/3000 [02:21<03:55,  8.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  33%|███▎      | 996/3000 [02:22<04:49,  6.92it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  33%|███▎      | 998/3000 [02:22<04:14,  7.87it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  33%|███▎      | 1001/3000 [02:22<03:21,  9.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  33%|███▎      | 1003/3000 [02:23<04:30,  7.38it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  33%|███▎      | 1004/3000 [02:23<06:52,  4.84it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  34%|███▎      | 1006/3000 [02:23<05:45,  5.77it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  34%|███▎      | 1009/3000 [02:24<05:43,  5.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  34%|███▎      | 1012/3000 [02:24<04:48,  6.88it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  34%|███▍      | 1014/3000 [02:24<04:28,  7.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  34%|███▍      | 1017/3000 [02:25<03:51,  8.58it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  34%|███▍      | 1019/3000 [02:25<03:53,  8.47it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  34%|███▍      | 1022/3000 [02:25<03:56,  8.35it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  34%|███▍      | 1025/3000 [02:25<02:55, 11.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  34%|███▍      | 1027/3000 [02:26<04:01,  8.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  34%|███▍      | 1030/3000 [02:26<04:46,  6.87it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  34%|███▍      | 1031/3000 [02:27<04:53,  6.72it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  34%|███▍      | 1032/3000 [02:27<07:31,  4.36it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  34%|███▍      | 1035/3000 [02:28<05:48,  5.63it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  35%|███▍      | 1037/3000 [02:28<05:11,  6.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  35%|███▍      | 1038/3000 [02:28<05:48,  5.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  35%|███▍      | 1040/3000 [02:28<05:11,  6.29it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  35%|███▍      | 1044/3000 [02:29<04:01,  8.09it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  35%|███▍      | 1045/3000 [02:29<03:54,  8.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  35%|███▍      | 1049/3000 [02:29<03:49,  8.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  35%|███▌      | 1052/3000 [02:29<03:09, 10.28it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  35%|███▌      | 1054/3000 [02:30<03:57,  8.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  35%|███▌      | 1055/3000 [02:30<04:54,  6.60it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  35%|███▌      | 1056/3000 [02:30<05:52,  5.52it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  35%|███▌      | 1059/3000 [02:31<05:42,  5.66it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  35%|███▌      | 1060/3000 [02:31<06:05,  5.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  35%|███▌      | 1061/3000 [02:31<06:44,  4.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  36%|███▌      | 1067/3000 [02:32<03:15,  9.86it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  36%|███▌      | 1069/3000 [02:32<03:13,  9.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  36%|███▌      | 1071/3000 [02:33<04:40,  6.87it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  36%|███▌      | 1073/3000 [02:33<04:07,  7.80it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  36%|███▌      | 1077/3000 [02:33<03:30,  9.16it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  36%|███▌      | 1079/3000 [02:33<03:00, 10.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  36%|███▌      | 1083/3000 [02:34<03:30,  9.09it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  36%|███▌      | 1085/3000 [02:34<04:55,  6.47it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  36%|███▋      | 1088/3000 [02:35<04:46,  6.67it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  36%|███▋      | 1091/3000 [02:35<04:06,  7.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  36%|███▋      | 1092/3000 [02:35<05:22,  5.92it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  36%|███▋      | 1094/3000 [02:36<05:32,  5.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  37%|███▋      | 1099/3000 [02:36<03:56,  8.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  37%|███▋      | 1104/3000 [02:37<02:43, 11.57it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  37%|███▋      | 1109/3000 [02:37<02:31, 12.48it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  37%|███▋      | 1111/3000 [02:37<03:08, 10.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  37%|███▋      | 1113/3000 [02:38<06:20,  4.96it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  37%|███▋      | 1116/3000 [02:38<04:23,  7.16it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  37%|███▋      | 1121/3000 [02:39<04:22,  7.17it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  38%|███▊      | 1126/3000 [02:39<03:14,  9.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  38%|███▊      | 1130/3000 [02:40<03:53,  7.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  38%|███▊      | 1134/3000 [02:40<03:26,  9.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  38%|███▊      | 1138/3000 [02:41<02:52, 10.80it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  38%|███▊      | 1140/3000 [02:41<03:14,  9.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  38%|███▊      | 1142/3000 [02:42<05:26,  5.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  38%|███▊      | 1145/3000 [02:42<04:42,  6.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  38%|███▊      | 1147/3000 [02:42<04:41,  6.57it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  38%|███▊      | 1151/3000 [02:43<03:15,  9.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  38%|███▊      | 1153/3000 [02:43<04:58,  6.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  39%|███▊      | 1156/3000 [02:43<03:55,  7.83it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  39%|███▊      | 1159/3000 [02:44<02:55, 10.48it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  39%|███▊      | 1161/3000 [02:44<03:07,  9.78it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  39%|███▉      | 1163/3000 [02:44<03:22,  9.06it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  39%|███▉      | 1166/3000 [02:44<02:32, 12.05it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  39%|███▉      | 1168/3000 [02:45<03:29,  8.74it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  39%|███▉      | 1170/3000 [02:45<03:54,  7.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  39%|███▉      | 1172/3000 [02:45<03:42,  8.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  39%|███▉      | 1174/3000 [02:46<05:20,  5.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  39%|███▉      | 1179/3000 [02:46<03:32,  8.58it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  39%|███▉      | 1181/3000 [02:46<03:07,  9.71it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  39%|███▉      | 1184/3000 [02:46<02:40, 11.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  40%|███▉      | 1186/3000 [02:47<04:20,  6.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  40%|███▉      | 1188/3000 [02:47<04:05,  7.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  40%|███▉      | 1193/3000 [02:48<03:59,  7.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  40%|███▉      | 1195/3000 [02:48<03:23,  8.87it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  40%|███▉      | 1197/3000 [02:48<03:13,  9.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  40%|███▉      | 1199/3000 [02:49<04:22,  6.87it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  40%|████      | 1204/3000 [02:50<04:24,  6.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  40%|████      | 1206/3000 [02:50<03:52,  7.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  40%|████      | 1211/3000 [02:50<02:56, 10.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  40%|████      | 1215/3000 [02:51<02:52, 10.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  41%|████      | 1218/3000 [02:51<04:55,  6.02it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  41%|████      | 1221/3000 [02:52<03:49,  7.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  41%|████      | 1223/3000 [02:52<03:16,  9.02it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  41%|████      | 1226/3000 [02:52<02:48, 10.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  41%|████      | 1228/3000 [02:53<03:58,  7.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  41%|████      | 1230/3000 [02:53<04:01,  7.32it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  41%|████      | 1231/3000 [02:53<04:37,  6.38it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  41%|████      | 1234/3000 [02:54<04:31,  6.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  41%|████      | 1236/3000 [02:54<04:11,  7.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  41%|████▏     | 1241/3000 [02:54<03:00,  9.72it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  41%|████▏     | 1243/3000 [02:54<02:47, 10.49it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  42%|████▏     | 1246/3000 [02:55<04:10,  7.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  42%|████▏     | 1248/3000 [02:55<04:27,  6.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  42%|████▏     | 1251/3000 [02:56<04:22,  6.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  42%|████▏     | 1252/3000 [02:56<04:38,  6.28it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  42%|████▏     | 1258/3000 [02:57<03:56,  7.37it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  42%|████▏     | 1263/3000 [02:58<03:31,  8.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  42%|████▏     | 1265/3000 [02:58<03:17,  8.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  42%|████▏     | 1267/3000 [02:58<04:28,  6.46it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  42%|████▏     | 1272/3000 [02:59<03:32,  8.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  42%|████▏     | 1274/3000 [02:59<04:29,  6.40it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  43%|████▎     | 1276/3000 [03:00<05:48,  4.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  43%|████▎     | 1277/3000 [03:00<05:51,  4.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  43%|████▎     | 1282/3000 [03:01<03:49,  7.49it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  43%|████▎     | 1284/3000 [03:01<03:59,  7.17it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  43%|████▎     | 1288/3000 [03:01<03:34,  7.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  43%|████▎     | 1289/3000 [03:02<04:19,  6.60it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  43%|████▎     | 1290/3000 [03:02<05:52,  4.86it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  43%|████▎     | 1294/3000 [03:03<04:11,  6.77it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  43%|████▎     | 1297/3000 [03:03<03:14,  8.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  43%|████▎     | 1299/3000 [03:03<03:56,  7.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  43%|████▎     | 1302/3000 [03:03<03:33,  7.97it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  43%|████▎     | 1303/3000 [03:04<04:25,  6.40it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  44%|████▎     | 1306/3000 [03:04<04:18,  6.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  44%|████▎     | 1308/3000 [03:04<03:31,  8.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  44%|████▎     | 1309/3000 [03:05<04:40,  6.02it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  44%|████▎     | 1310/3000 [03:05<06:04,  4.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  44%|████▍     | 1313/3000 [03:05<03:59,  7.04it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  44%|████▍     | 1314/3000 [03:06<04:45,  5.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  44%|████▍     | 1317/3000 [03:06<03:56,  7.11it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  44%|████▍     | 1320/3000 [03:06<03:40,  7.61it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  44%|████▍     | 1322/3000 [03:07<03:51,  7.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  44%|████▍     | 1325/3000 [03:07<04:17,  6.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  44%|████▍     | 1329/3000 [03:08<03:25,  8.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  44%|████▍     | 1332/3000 [03:08<02:50,  9.80it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  44%|████▍     | 1334/3000 [03:08<03:36,  7.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  44%|████▍     | 1335/3000 [03:09<04:48,  5.78it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  45%|████▍     | 1339/3000 [03:09<03:49,  7.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  45%|████▍     | 1341/3000 [03:09<03:10,  8.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  45%|████▍     | 1343/3000 [03:10<04:21,  6.35it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  45%|████▍     | 1345/3000 [03:10<03:47,  7.27it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  45%|████▌     | 1350/3000 [03:10<02:43, 10.06it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  45%|████▌     | 1352/3000 [03:11<04:14,  6.48it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  45%|████▌     | 1353/3000 [03:11<05:07,  5.36it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  45%|████▌     | 1356/3000 [03:11<03:55,  6.97it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  45%|████▌     | 1359/3000 [03:12<04:02,  6.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  45%|████▌     | 1360/3000 [03:12<04:37,  5.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  45%|████▌     | 1361/3000 [03:12<04:52,  5.60it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  45%|████▌     | 1364/3000 [03:13<04:17,  6.36it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  46%|████▌     | 1366/3000 [03:13<03:26,  7.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  46%|████▌     | 1370/3000 [03:13<02:26, 11.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  46%|████▌     | 1372/3000 [03:13<02:35, 10.49it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  46%|████▌     | 1374/3000 [03:14<03:49,  7.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  46%|████▌     | 1376/3000 [03:15<05:18,  5.09it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  46%|████▌     | 1377/3000 [03:15<04:48,  5.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  46%|████▌     | 1380/3000 [03:15<05:02,  5.35it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  46%|████▌     | 1384/3000 [03:16<03:32,  7.61it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  46%|████▌     | 1385/3000 [03:16<03:23,  7.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  46%|████▌     | 1386/3000 [03:16<04:55,  5.47it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  46%|████▋     | 1388/3000 [03:17<04:51,  5.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  46%|████▋     | 1391/3000 [03:17<03:28,  7.71it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  46%|████▋     | 1394/3000 [03:17<02:19, 11.48it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  47%|████▋     | 1396/3000 [03:17<02:17, 11.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  47%|████▋     | 1400/3000 [03:18<04:31,  5.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  47%|████▋     | 1402/3000 [03:19<03:35,  7.42it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  47%|████▋     | 1404/3000 [03:19<03:30,  7.57it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  47%|████▋     | 1406/3000 [03:19<03:10,  8.37it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  47%|████▋     | 1408/3000 [03:19<03:42,  7.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  47%|████▋     | 1410/3000 [03:20<04:18,  6.16it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  47%|████▋     | 1411/3000 [03:20<05:11,  5.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  47%|████▋     | 1415/3000 [03:21<03:56,  6.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  47%|████▋     | 1418/3000 [03:21<04:07,  6.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  47%|████▋     | 1419/3000 [03:21<05:12,  5.05it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  47%|████▋     | 1424/3000 [03:22<03:09,  8.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  48%|████▊     | 1427/3000 [03:22<02:30, 10.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  48%|████▊     | 1430/3000 [03:23<04:38,  5.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  48%|████▊     | 1432/3000 [03:23<04:03,  6.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  48%|████▊     | 1435/3000 [03:23<03:08,  8.29it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  48%|████▊     | 1437/3000 [03:24<04:18,  6.04it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  48%|████▊     | 1439/3000 [03:24<04:06,  6.32it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  48%|████▊     | 1442/3000 [03:25<04:06,  6.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  48%|████▊     | 1446/3000 [03:25<03:55,  6.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  48%|████▊     | 1450/3000 [03:26<03:54,  6.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  48%|████▊     | 1451/3000 [03:26<04:10,  6.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  48%|████▊     | 1455/3000 [03:27<03:36,  7.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  49%|████▊     | 1457/3000 [03:27<03:15,  7.88it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  49%|████▊     | 1459/3000 [03:27<02:56,  8.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  49%|████▊     | 1462/3000 [03:27<03:39,  7.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  49%|████▉     | 1464/3000 [03:28<04:14,  6.04it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  49%|████▉     | 1466/3000 [03:28<03:33,  7.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  49%|████▉     | 1467/3000 [03:29<04:41,  5.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  49%|████▉     | 1471/3000 [03:29<03:14,  7.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  49%|████▉     | 1474/3000 [03:29<02:55,  8.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  49%|████▉     | 1476/3000 [03:30<03:19,  7.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  49%|████▉     | 1477/3000 [03:30<04:32,  5.58it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  49%|████▉     | 1480/3000 [03:30<03:47,  6.67it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  49%|████▉     | 1482/3000 [03:30<03:23,  7.47it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  49%|████▉     | 1484/3000 [03:31<03:28,  7.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  50%|████▉     | 1486/3000 [03:31<03:13,  7.82it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  50%|████▉     | 1490/3000 [03:31<02:32,  9.88it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  50%|████▉     | 1493/3000 [03:32<03:23,  7.42it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  50%|████▉     | 1494/3000 [03:32<03:40,  6.84it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  50%|████▉     | 1495/3000 [03:32<04:18,  5.83it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  50%|████▉     | 1496/3000 [03:33<05:05,  4.92it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  50%|████▉     | 1499/3000 [03:33<04:25,  5.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  50%|█████     | 1503/3000 [03:33<02:55,  8.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  50%|█████     | 1505/3000 [03:34<02:25, 10.27it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  50%|█████     | 1509/3000 [03:34<02:23, 10.37it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  50%|█████     | 1511/3000 [03:34<03:36,  6.88it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  50%|█████     | 1513/3000 [03:35<03:50,  6.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  51%|█████     | 1520/3000 [03:35<02:33,  9.63it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  51%|█████     | 1522/3000 [03:36<02:20, 10.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  51%|█████     | 1524/3000 [03:36<03:11,  7.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  51%|█████     | 1526/3000 [03:36<03:12,  7.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  51%|█████     | 1527/3000 [03:37<03:43,  6.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  51%|█████     | 1530/3000 [03:37<05:03,  4.84it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  51%|█████     | 1536/3000 [03:38<03:14,  7.52it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  51%|█████▏    | 1539/3000 [03:39<03:54,  6.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  51%|█████▏    | 1541/3000 [03:39<03:17,  7.37it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  51%|█████▏    | 1544/3000 [03:39<03:11,  7.61it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  52%|█████▏    | 1547/3000 [03:39<02:47,  8.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  52%|█████▏    | 1553/3000 [03:40<01:52, 12.89it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  52%|█████▏    | 1555/3000 [03:41<03:54,  6.17it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  52%|█████▏    | 1557/3000 [03:41<04:26,  5.42it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  52%|█████▏    | 1561/3000 [03:41<03:05,  7.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  52%|█████▏    | 1563/3000 [03:42<03:08,  7.61it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  52%|█████▏    | 1566/3000 [03:42<03:06,  7.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  52%|█████▏    | 1567/3000 [03:42<03:28,  6.87it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  52%|█████▏    | 1570/3000 [03:43<03:03,  7.78it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  52%|█████▏    | 1573/3000 [03:43<03:01,  7.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  52%|█████▎    | 1575/3000 [03:43<03:21,  7.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  53%|█████▎    | 1577/3000 [03:44<03:50,  6.17it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  53%|█████▎    | 1580/3000 [03:44<04:28,  5.29it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  53%|█████▎    | 1582/3000 [03:45<03:34,  6.61it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  53%|█████▎    | 1587/3000 [03:45<02:17, 10.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  53%|█████▎    | 1591/3000 [03:45<02:31,  9.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  53%|█████▎    | 1594/3000 [03:46<03:13,  7.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  53%|█████▎    | 1595/3000 [03:46<03:51,  6.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  53%|█████▎    | 1596/3000 [03:46<04:09,  5.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  53%|█████▎    | 1601/3000 [03:47<02:30,  9.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  53%|█████▎    | 1603/3000 [03:47<02:08, 10.87it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  54%|█████▎    | 1605/3000 [03:48<03:37,  6.42it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  54%|█████▎    | 1606/3000 [03:48<04:18,  5.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  54%|█████▎    | 1607/3000 [03:48<04:31,  5.14it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  54%|█████▍    | 1613/3000 [03:49<02:55,  7.92it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  54%|█████▍    | 1616/3000 [03:49<02:14, 10.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  54%|█████▍    | 1621/3000 [03:49<02:00, 11.42it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  54%|█████▍    | 1623/3000 [03:49<01:58, 11.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  54%|█████▍    | 1625/3000 [03:50<02:44,  8.35it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  54%|█████▍    | 1627/3000 [03:51<04:40,  4.89it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  54%|█████▍    | 1631/3000 [03:51<03:49,  5.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  54%|█████▍    | 1633/3000 [03:51<03:21,  6.78it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  55%|█████▍    | 1636/3000 [03:52<02:51,  7.96it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  55%|█████▍    | 1641/3000 [03:52<02:14, 10.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  55%|█████▍    | 1643/3000 [03:52<02:26,  9.27it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  55%|█████▍    | 1645/3000 [03:53<02:55,  7.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  55%|█████▍    | 1647/3000 [03:53<02:47,  8.07it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  55%|█████▍    | 1649/3000 [03:53<03:43,  6.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  55%|█████▌    | 1652/3000 [03:54<03:51,  5.83it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  55%|█████▌    | 1656/3000 [03:54<03:17,  6.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  55%|█████▌    | 1658/3000 [03:55<03:22,  6.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  55%|█████▌    | 1659/3000 [03:55<04:07,  5.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  55%|█████▌    | 1662/3000 [03:56<03:31,  6.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  56%|█████▌    | 1665/3000 [03:56<02:29,  8.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  56%|█████▌    | 1667/3000 [03:56<02:25,  9.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  56%|█████▌    | 1669/3000 [03:56<02:06, 10.48it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  56%|█████▌    | 1674/3000 [03:56<01:53, 11.71it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  56%|█████▌    | 1676/3000 [03:57<02:02, 10.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  56%|█████▌    | 1680/3000 [03:58<03:43,  5.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  56%|█████▌    | 1683/3000 [03:58<03:06,  7.05it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  56%|█████▌    | 1685/3000 [03:58<03:31,  6.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  56%|█████▋    | 1689/3000 [03:59<03:16,  6.66it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  56%|█████▋    | 1694/3000 [04:00<02:27,  8.83it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  57%|█████▋    | 1697/3000 [04:00<02:01, 10.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  57%|█████▋    | 1700/3000 [04:00<01:41, 12.78it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  57%|█████▋    | 1702/3000 [04:00<02:09, 10.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  57%|█████▋    | 1704/3000 [04:00<02:04, 10.38it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  57%|█████▋    | 1706/3000 [04:01<02:11,  9.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  57%|█████▋    | 1708/3000 [04:01<03:39,  5.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  57%|█████▋    | 1709/3000 [04:02<04:05,  5.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  57%|█████▋    | 1710/3000 [04:02<04:55,  4.36it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  57%|█████▋    | 1713/3000 [04:02<03:55,  5.46it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  57%|█████▋    | 1717/3000 [04:03<02:48,  7.60it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  57%|█████▋    | 1719/3000 [04:03<02:16,  9.37it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  57%|█████▋    | 1722/3000 [04:04<03:10,  6.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  57%|█████▋    | 1724/3000 [04:04<02:28,  8.57it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  58%|█████▊    | 1726/3000 [04:04<02:48,  7.55it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  58%|█████▊    | 1730/3000 [04:04<02:13,  9.52it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  58%|█████▊    | 1732/3000 [04:04<02:02, 10.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  58%|█████▊    | 1736/3000 [04:05<02:39,  7.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  58%|█████▊    | 1739/3000 [04:06<03:47,  5.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  58%|█████▊    | 1742/3000 [04:06<03:04,  6.83it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  58%|█████▊    | 1743/3000 [04:07<03:21,  6.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  58%|█████▊    | 1745/3000 [04:07<02:54,  7.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  58%|█████▊    | 1746/3000 [04:07<03:42,  5.63it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  58%|█████▊    | 1749/3000 [04:07<03:08,  6.63it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  58%|█████▊    | 1750/3000 [04:08<04:05,  5.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  59%|█████▊    | 1756/3000 [04:08<02:18,  9.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  59%|█████▊    | 1759/3000 [04:08<01:51, 11.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  59%|█████▊    | 1761/3000 [04:09<02:46,  7.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  59%|█████▊    | 1762/3000 [04:09<03:06,  6.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  59%|█████▉    | 1763/3000 [04:09<03:39,  5.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  59%|█████▉    | 1767/3000 [04:10<02:45,  7.46it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  59%|█████▉    | 1768/3000 [04:10<02:46,  7.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  59%|█████▉    | 1770/3000 [04:10<03:30,  5.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  59%|█████▉    | 1771/3000 [04:11<05:01,  4.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  59%|█████▉    | 1776/3000 [04:11<02:37,  7.77it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  59%|█████▉    | 1778/3000 [04:12<03:00,  6.78it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metricsmetrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      " {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  59%|█████▉    | 1780/3000 [04:12<03:12,  6.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  59%|█████▉    | 1781/3000 [04:12<03:41,  5.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  60%|█████▉    | 1786/3000 [04:13<02:38,  7.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  60%|█████▉    | 1789/3000 [04:13<02:51,  7.04it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  60%|█████▉    | 1790/3000 [04:13<02:46,  7.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  60%|█████▉    | 1792/3000 [04:14<04:11,  4.80it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  60%|█████▉    | 1796/3000 [04:14<02:17,  8.74it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  60%|█████▉    | 1798/3000 [04:15<02:49,  7.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  60%|██████    | 1802/3000 [04:15<02:28,  8.07it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  60%|██████    | 1804/3000 [04:15<02:14,  8.87it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  60%|██████    | 1806/3000 [04:16<03:23,  5.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  60%|██████    | 1809/3000 [04:17<03:06,  6.40it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  60%|██████    | 1810/3000 [04:17<02:58,  6.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  60%|██████    | 1814/3000 [04:17<03:08,  6.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  60%|██████    | 1815/3000 [04:18<03:45,  5.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  61%|██████    | 1816/3000 [04:18<03:52,  5.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  61%|██████    | 1821/3000 [04:18<02:27,  7.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  61%|██████    | 1824/3000 [04:19<02:08,  9.14it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  61%|██████    | 1825/3000 [04:19<02:38,  7.42it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  61%|██████    | 1826/3000 [04:19<03:11,  6.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  61%|██████    | 1827/3000 [04:19<04:02,  4.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  61%|██████    | 1829/3000 [04:20<03:51,  5.06it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  61%|██████    | 1831/3000 [04:20<03:08,  6.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  61%|██████    | 1834/3000 [04:20<02:34,  7.55it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  61%|██████▏   | 1838/3000 [04:21<01:41, 11.48it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  61%|██████▏   | 1840/3000 [04:21<03:24,  5.67it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  61%|██████▏   | 1844/3000 [04:22<03:13,  5.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  62%|██████▏   | 1846/3000 [04:22<02:49,  6.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  62%|██████▏   | 1851/3000 [04:23<02:29,  7.71it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  62%|██████▏   | 1853/3000 [04:23<02:09,  8.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  62%|██████▏   | 1854/3000 [04:23<02:53,  6.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  62%|██████▏   | 1857/3000 [04:24<02:58,  6.40it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  62%|██████▏   | 1862/3000 [04:24<01:51, 10.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  62%|██████▏   | 1864/3000 [04:24<02:07,  8.93it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  62%|██████▏   | 1866/3000 [04:25<02:13,  8.48it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  62%|██████▏   | 1868/3000 [04:26<04:16,  4.42it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  62%|██████▏   | 1870/3000 [04:26<04:13,  4.46it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  62%|██████▏   | 1871/3000 [04:26<04:19,  4.35it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  62%|██████▏   | 1873/3000 [04:27<03:27,  5.42it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  62%|██████▎   | 1875/3000 [04:27<02:54,  6.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  63%|██████▎   | 1878/3000 [04:27<02:26,  7.66it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  63%|██████▎   | 1881/3000 [04:27<02:27,  7.58it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  63%|██████▎   | 1884/3000 [04:28<02:05,  8.88it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  63%|██████▎   | 1886/3000 [04:28<02:52,  6.46it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  63%|██████▎   | 1889/3000 [04:28<01:51,  9.97it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  63%|██████▎   | 1891/3000 [04:29<02:37,  7.05it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  63%|██████▎   | 1896/3000 [04:29<02:04,  8.86it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  63%|██████▎   | 1898/3000 [04:30<02:12,  8.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  63%|██████▎   | 1900/3000 [04:31<04:16,  4.29it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  63%|██████▎   | 1903/3000 [04:31<03:18,  5.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  64%|██████▎   | 1908/3000 [04:31<02:11,  8.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  64%|██████▎   | 1911/3000 [04:32<02:08,  8.46it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  64%|██████▍   | 1913/3000 [04:32<03:35,  5.05it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  64%|██████▍   | 1917/3000 [04:33<02:59,  6.05it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  64%|██████▍   | 1921/3000 [04:33<02:24,  7.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  64%|██████▍   | 1925/3000 [04:34<02:04,  8.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  64%|██████▍   | 1927/3000 [04:34<02:10,  8.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  64%|██████▍   | 1930/3000 [04:35<03:11,  5.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  64%|██████▍   | 1931/3000 [04:35<02:58,  6.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  65%|██████▍   | 1938/3000 [04:36<01:43, 10.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  65%|██████▍   | 1940/3000 [04:36<01:49,  9.71it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  65%|██████▍   | 1942/3000 [04:37<03:06,  5.67it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  65%|██████▍   | 1944/3000 [04:37<03:19,  5.28it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  65%|██████▍   | 1947/3000 [04:37<02:30,  7.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  65%|██████▍   | 1949/3000 [04:38<02:52,  6.09it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  65%|██████▌   | 1953/3000 [04:38<02:08,  8.16it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  65%|██████▌   | 1956/3000 [04:38<01:49,  9.57it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  65%|██████▌   | 1960/3000 [04:39<02:34,  6.72it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  65%|██████▌   | 1964/3000 [04:40<02:16,  7.58it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  66%|██████▌   | 1966/3000 [04:40<02:03,  8.40it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  66%|██████▌   | 1968/3000 [04:40<02:28,  6.93it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  66%|██████▌   | 1971/3000 [04:41<02:27,  6.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  66%|██████▌   | 1973/3000 [04:41<02:47,  6.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  66%|██████▌   | 1977/3000 [04:42<02:23,  7.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  66%|██████▌   | 1981/3000 [04:42<02:49,  6.02it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  66%|██████▌   | 1983/3000 [04:43<02:16,  7.47it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  66%|██████▌   | 1987/3000 [04:43<01:48,  9.35it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  66%|██████▋   | 1991/3000 [04:44<02:12,  7.60it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  66%|██████▋   | 1995/3000 [04:44<02:04,  8.07it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  67%|██████▋   | 1997/3000 [04:45<02:41,  6.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  67%|██████▋   | 2000/3000 [04:45<02:03,  8.11it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  67%|██████▋   | 2004/3000 [04:45<01:57,  8.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  67%|██████▋   | 2006/3000 [04:46<02:14,  7.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  67%|██████▋   | 2009/3000 [04:46<02:08,  7.72it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  67%|██████▋   | 2011/3000 [04:46<02:28,  6.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  67%|██████▋   | 2014/3000 [04:47<02:03,  8.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  67%|██████▋   | 2017/3000 [04:47<02:13,  7.37it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  67%|██████▋   | 2018/3000 [04:47<02:08,  7.66it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  67%|██████▋   | 2019/3000 [04:48<02:42,  6.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  67%|██████▋   | 2022/3000 [04:48<02:38,  6.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  67%|██████▋   | 2024/3000 [04:48<02:07,  7.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  68%|██████▊   | 2025/3000 [04:48<02:10,  7.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  68%|██████▊   | 2028/3000 [04:49<02:11,  7.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  68%|██████▊   | 2030/3000 [04:49<02:27,  6.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  68%|██████▊   | 2031/3000 [04:50<03:07,  5.16it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  68%|██████▊   | 2034/3000 [04:50<02:36,  6.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  68%|██████▊   | 2036/3000 [04:50<02:25,  6.63it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  68%|██████▊   | 2039/3000 [04:51<02:26,  6.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  68%|██████▊   | 2042/3000 [04:51<02:15,  7.09it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  68%|██████▊   | 2044/3000 [04:51<01:50,  8.67it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  68%|██████▊   | 2046/3000 [04:52<02:31,  6.29it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  68%|██████▊   | 2051/3000 [04:52<01:49,  8.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  68%|██████▊   | 2055/3000 [04:53<02:11,  7.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  69%|██████▊   | 2056/3000 [04:53<02:25,  6.49it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  69%|██████▊   | 2060/3000 [04:54<02:15,  6.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  69%|██████▊   | 2062/3000 [04:54<02:19,  6.74it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  69%|██████▉   | 2065/3000 [04:54<02:18,  6.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  69%|██████▉   | 2068/3000 [04:55<02:11,  7.07it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  69%|██████▉   | 2070/3000 [04:55<02:01,  7.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  69%|██████▉   | 2072/3000 [04:55<02:30,  6.17it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  69%|██████▉   | 2076/3000 [04:56<01:48,  8.49it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  69%|██████▉   | 2077/3000 [04:56<01:49,  8.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  69%|██████▉   | 2081/3000 [04:56<01:40,  9.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  69%|██████▉   | 2083/3000 [04:57<01:50,  8.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  70%|██████▉   | 2085/3000 [04:57<01:58,  7.74it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  70%|██████▉   | 2086/3000 [04:57<02:45,  5.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  70%|██████▉   | 2087/3000 [04:58<03:10,  4.80it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  70%|██████▉   | 2091/3000 [04:58<02:10,  6.97it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  70%|██████▉   | 2096/3000 [04:59<01:37,  9.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  70%|██████▉   | 2098/3000 [04:59<02:03,  7.27it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  70%|███████   | 2101/3000 [04:59<02:14,  6.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  70%|███████   | 2104/3000 [05:00<02:23,  6.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  70%|███████   | 2109/3000 [05:01<01:55,  7.74it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  70%|███████   | 2112/3000 [05:01<01:30,  9.82it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  70%|███████   | 2114/3000 [05:01<01:18, 11.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  71%|███████   | 2116/3000 [05:01<01:55,  7.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  71%|███████   | 2118/3000 [05:02<01:59,  7.38it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  71%|███████   | 2119/3000 [05:02<02:22,  6.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  71%|███████   | 2120/3000 [05:02<02:47,  5.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  71%|███████   | 2123/3000 [05:03<02:03,  7.09it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  71%|███████   | 2125/3000 [05:03<02:00,  7.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  71%|███████   | 2131/3000 [05:03<01:23, 10.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  71%|███████   | 2133/3000 [05:04<01:43,  8.37it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  71%|███████   | 2134/3000 [05:04<01:57,  7.35it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  71%|███████   | 2135/3000 [05:04<03:08,  4.60it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  71%|███████▏  | 2138/3000 [05:05<02:35,  5.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  71%|███████▏  | 2140/3000 [05:05<01:56,  7.38it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  71%|███████▏  | 2142/3000 [05:05<01:54,  7.46it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  71%|███████▏  | 2144/3000 [05:05<01:43,  8.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  72%|███████▏  | 2145/3000 [05:06<02:25,  5.86it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  72%|███████▏  | 2149/3000 [05:06<01:52,  7.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  72%|███████▏  | 2151/3000 [05:07<02:15,  6.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  72%|███████▏  | 2153/3000 [05:07<02:07,  6.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  72%|███████▏  | 2156/3000 [05:07<01:52,  7.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  72%|███████▏  | 2159/3000 [05:07<01:20, 10.38it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  72%|███████▏  | 2163/3000 [05:08<01:40,  8.36it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  72%|███████▏  | 2165/3000 [05:08<01:45,  7.88it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  72%|███████▏  | 2168/3000 [05:09<01:53,  7.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  72%|███████▏  | 2171/3000 [05:09<01:48,  7.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  72%|███████▏  | 2173/3000 [05:10<01:58,  6.96it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  73%|███████▎  | 2176/3000 [05:10<02:08,  6.40it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  73%|███████▎  | 2177/3000 [05:10<02:05,  6.55it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  73%|███████▎  | 2181/3000 [05:11<01:42,  8.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  73%|███████▎  | 2183/3000 [05:11<01:54,  7.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  73%|███████▎  | 2187/3000 [05:12<01:43,  7.87it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  73%|███████▎  | 2189/3000 [05:12<02:04,  6.52it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  73%|███████▎  | 2193/3000 [05:12<01:17, 10.36it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  73%|███████▎  | 2197/3000 [05:13<01:38,  8.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  73%|███████▎  | 2200/3000 [05:13<01:41,  7.92it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  73%|███████▎  | 2201/3000 [05:14<02:02,  6.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  73%|███████▎  | 2203/3000 [05:14<02:33,  5.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  74%|███████▎  | 2206/3000 [05:14<01:35,  8.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  74%|███████▎  | 2208/3000 [05:14<01:22,  9.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  74%|███████▎  | 2210/3000 [05:15<01:34,  8.32it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  74%|███████▍  | 2214/3000 [05:16<02:06,  6.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  74%|███████▍  | 2216/3000 [05:16<01:42,  7.63it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  74%|███████▍  | 2218/3000 [05:16<02:02,  6.38it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  74%|███████▍  | 2222/3000 [05:17<01:33,  8.28it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  74%|███████▍  | 2226/3000 [05:17<01:15, 10.29it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  74%|███████▍  | 2229/3000 [05:18<02:11,  5.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  74%|███████▍  | 2235/3000 [05:18<01:36,  7.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  75%|███████▍  | 2237/3000 [05:19<01:26,  8.87it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  75%|███████▍  | 2239/3000 [05:19<01:37,  7.77it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  75%|███████▍  | 2241/3000 [05:19<01:44,  7.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  75%|███████▍  | 2242/3000 [05:19<01:46,  7.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  75%|███████▍  | 2243/3000 [05:20<02:34,  4.89it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  75%|███████▍  | 2244/3000 [05:20<02:46,  4.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  75%|███████▍  | 2249/3000 [05:20<01:31,  8.17it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  75%|███████▌  | 2252/3000 [05:21<01:35,  7.83it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  75%|███████▌  | 2255/3000 [05:21<01:11, 10.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  75%|███████▌  | 2257/3000 [05:22<01:56,  6.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  75%|███████▌  | 2258/3000 [05:22<02:21,  5.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  75%|███████▌  | 2263/3000 [05:22<01:27,  8.46it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  76%|███████▌  | 2267/3000 [05:23<01:21,  9.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  76%|███████▌  | 2269/3000 [05:23<01:22,  8.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  76%|███████▌  | 2271/3000 [05:23<01:19,  9.17it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  76%|███████▌  | 2274/3000 [05:24<01:56,  6.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  76%|███████▌  | 2275/3000 [05:24<02:15,  5.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  76%|███████▌  | 2281/3000 [05:25<01:15,  9.57it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  76%|███████▌  | 2285/3000 [05:25<01:00, 11.78it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  76%|███████▌  | 2287/3000 [05:25<01:16,  9.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  76%|███████▋  | 2289/3000 [05:26<02:11,  5.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  76%|███████▋  | 2292/3000 [05:26<01:48,  6.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  76%|███████▋  | 2294/3000 [05:27<01:43,  6.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  77%|███████▋  | 2297/3000 [05:27<01:46,  6.60it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  77%|███████▋  | 2301/3000 [05:28<01:19,  8.78it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  77%|███████▋  | 2303/3000 [05:28<01:34,  7.38it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  77%|███████▋  | 2305/3000 [05:28<01:44,  6.63it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  77%|███████▋  | 2309/3000 [05:29<01:14,  9.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  77%|███████▋  | 2311/3000 [05:29<01:06, 10.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  77%|███████▋  | 2313/3000 [05:29<01:07, 10.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  77%|███████▋  | 2316/3000 [05:29<01:27,  7.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  77%|███████▋  | 2318/3000 [05:30<02:10,  5.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  77%|███████▋  | 2321/3000 [05:30<01:48,  6.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  77%|███████▋  | 2323/3000 [05:31<01:50,  6.14it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  78%|███████▊  | 2326/3000 [05:31<01:24,  7.97it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  78%|███████▊  | 2329/3000 [05:31<01:18,  8.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  78%|███████▊  | 2330/3000 [05:32<01:44,  6.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  78%|███████▊  | 2333/3000 [05:32<01:47,  6.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  78%|███████▊  | 2337/3000 [05:33<01:31,  7.27it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  78%|███████▊  | 2340/3000 [05:33<01:12,  9.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  78%|███████▊  | 2343/3000 [05:33<00:58, 11.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  78%|███████▊  | 2347/3000 [05:34<01:22,  7.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  78%|███████▊  | 2349/3000 [05:35<02:09,  5.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  78%|███████▊  | 2352/3000 [05:35<01:45,  6.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  78%|███████▊  | 2355/3000 [05:35<01:44,  6.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  79%|███████▊  | 2356/3000 [05:36<01:54,  5.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  79%|███████▊  | 2359/3000 [05:36<01:27,  7.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  79%|███████▊  | 2360/3000 [05:36<01:51,  5.72it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  79%|███████▊  | 2362/3000 [05:37<01:46,  5.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  79%|███████▉  | 2364/3000 [05:37<01:33,  6.82it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  79%|███████▉  | 2368/3000 [05:37<01:01, 10.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  79%|███████▉  | 2371/3000 [05:37<01:01, 10.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  79%|███████▉  | 2373/3000 [05:38<01:37,  6.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  79%|███████▉  | 2375/3000 [05:39<02:00,  5.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  79%|███████▉  | 2380/3000 [05:39<01:16,  8.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metricsmetrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      " {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  79%|███████▉  | 2382/3000 [05:39<01:30,  6.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  79%|███████▉  | 2384/3000 [05:40<01:17,  7.97it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  80%|███████▉  | 2387/3000 [05:40<01:18,  7.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  80%|███████▉  | 2390/3000 [05:41<01:42,  5.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  80%|███████▉  | 2394/3000 [05:41<01:40,  6.06it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  80%|███████▉  | 2397/3000 [05:42<01:14,  8.14it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  80%|███████▉  | 2398/3000 [05:42<01:14,  8.11it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  80%|███████▉  | 2399/3000 [05:42<01:37,  6.16it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  80%|████████  | 2401/3000 [05:42<01:31,  6.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  80%|████████  | 2403/3000 [05:43<01:39,  5.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  80%|████████  | 2404/3000 [05:43<01:30,  6.55it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  80%|████████  | 2408/3000 [05:43<01:19,  7.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  80%|████████  | 2410/3000 [05:44<01:13,  8.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  80%|████████  | 2412/3000 [05:44<01:43,  5.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  80%|████████  | 2413/3000 [05:44<01:34,  6.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  80%|████████  | 2414/3000 [05:45<01:40,  5.82it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  81%|████████  | 2416/3000 [05:45<01:48,  5.40it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  81%|████████  | 2418/3000 [05:45<01:39,  5.83it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  81%|████████  | 2419/3000 [05:45<01:44,  5.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  81%|████████  | 2425/3000 [05:46<00:54, 10.47it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  81%|████████  | 2427/3000 [05:46<01:13,  7.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  81%|████████  | 2429/3000 [05:47<01:40,  5.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  81%|████████  | 2430/3000 [05:47<01:57,  4.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  81%|████████  | 2432/3000 [05:47<01:41,  5.57it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  81%|████████  | 2434/3000 [05:48<01:42,  5.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  81%|████████  | 2437/3000 [05:48<01:25,  6.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  81%|████████▏ | 2441/3000 [05:49<01:06,  8.36it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  81%|████████▏ | 2443/3000 [05:49<01:45,  5.27it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  82%|████████▏ | 2447/3000 [05:50<01:04,  8.55it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  82%|████████▏ | 2449/3000 [05:50<01:04,  8.58it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  82%|████████▏ | 2451/3000 [05:50<01:06,  8.32it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  82%|████████▏ | 2454/3000 [05:50<00:48, 11.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  82%|████████▏ | 2456/3000 [05:51<01:26,  6.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  82%|████████▏ | 2459/3000 [05:51<01:30,  5.96it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  82%|████████▏ | 2462/3000 [05:52<01:15,  7.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  82%|████████▏ | 2464/3000 [05:52<01:32,  5.77it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  82%|████████▏ | 2465/3000 [05:53<01:35,  5.61it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  82%|████████▏ | 2467/3000 [05:53<01:27,  6.09it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  82%|████████▏ | 2468/3000 [05:53<01:39,  5.35it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  82%|████████▏ | 2471/3000 [05:53<01:16,  6.93it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  82%|████████▏ | 2473/3000 [05:54<01:11,  7.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  82%|████████▎ | 2475/3000 [05:54<01:17,  6.77it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  83%|████████▎ | 2476/3000 [05:54<01:13,  7.14it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  83%|████████▎ | 2478/3000 [05:54<01:24,  6.16it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  83%|████████▎ | 2480/3000 [05:55<01:31,  5.71it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  83%|████████▎ | 2485/3000 [05:55<01:02,  8.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  83%|████████▎ | 2487/3000 [05:55<00:52,  9.80it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  83%|████████▎ | 2489/3000 [05:56<01:28,  5.80it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  83%|████████▎ | 2490/3000 [05:56<01:39,  5.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  83%|████████▎ | 2492/3000 [05:57<01:49,  4.66it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  83%|████████▎ | 2493/3000 [05:57<02:02,  4.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  83%|████████▎ | 2498/3000 [05:58<01:33,  5.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  83%|████████▎ | 2500/3000 [05:58<01:16,  6.55it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  83%|████████▎ | 2504/3000 [05:59<01:22,  5.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  84%|████████▎ | 2506/3000 [05:59<01:09,  7.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  84%|████████▎ | 2507/3000 [05:59<01:22,  5.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  84%|████████▎ | 2509/3000 [06:00<01:25,  5.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  84%|████████▎ | 2511/3000 [06:00<01:27,  5.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  84%|████████▍ | 2513/3000 [06:00<01:15,  6.46it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  84%|████████▍ | 2515/3000 [06:01<01:14,  6.49it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  84%|████████▍ | 2519/3000 [06:01<00:51,  9.28it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  84%|████████▍ | 2524/3000 [06:01<00:38, 12.36it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  84%|████████▍ | 2526/3000 [06:02<01:03,  7.48it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  84%|████████▍ | 2528/3000 [06:02<01:09,  6.83it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  84%|████████▍ | 2529/3000 [06:03<01:20,  5.87it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  84%|████████▍ | 2532/3000 [06:03<01:17,  6.06it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  84%|████████▍ | 2535/3000 [06:03<00:59,  7.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  85%|████████▍ | 2536/3000 [06:03<00:58,  7.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  85%|████████▍ | 2537/3000 [06:04<01:14,  6.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  85%|████████▍ | 2539/3000 [06:04<01:13,  6.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  85%|████████▍ | 2540/3000 [06:04<01:25,  5.35it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  85%|████████▍ | 2542/3000 [06:05<01:23,  5.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  85%|████████▍ | 2543/3000 [06:05<01:25,  5.35it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  85%|████████▍ | 2544/3000 [06:05<01:27,  5.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  85%|████████▍ | 2549/3000 [06:06<00:54,  8.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  85%|████████▌ | 2552/3000 [06:06<00:39, 11.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  85%|████████▌ | 2554/3000 [06:06<00:41, 10.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  85%|████████▌ | 2556/3000 [06:07<01:09,  6.38it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  85%|████████▌ | 2557/3000 [06:07<01:16,  5.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  85%|████████▌ | 2559/3000 [06:07<01:10,  6.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  85%|████████▌ | 2562/3000 [06:07<01:09,  6.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  85%|████████▌ | 2564/3000 [06:08<01:01,  7.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  86%|████████▌ | 2566/3000 [06:08<00:57,  7.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  86%|████████▌ | 2569/3000 [06:08<00:36, 11.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  86%|████████▌ | 2571/3000 [06:09<01:22,  5.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  86%|████████▌ | 2574/3000 [06:09<01:11,  5.93it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  86%|████████▌ | 2577/3000 [06:10<01:16,  5.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  86%|████████▌ | 2579/3000 [06:10<00:59,  7.07it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  86%|████████▌ | 2582/3000 [06:11<01:03,  6.55it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  86%|████████▌ | 2585/3000 [06:11<00:47,  8.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  86%|████████▌ | 2587/3000 [06:11<00:47,  8.74it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  86%|████████▋ | 2592/3000 [06:11<00:35, 11.57it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  86%|████████▋ | 2594/3000 [06:12<01:05,  6.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  87%|████████▋ | 2596/3000 [06:13<01:22,  4.92it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  87%|████████▋ | 2598/3000 [06:13<01:13,  5.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  87%|████████▋ | 2599/3000 [06:13<01:08,  5.87it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  87%|████████▋ | 2600/3000 [06:13<01:25,  4.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  87%|████████▋ | 2603/3000 [06:14<01:23,  4.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  87%|████████▋ | 2607/3000 [06:14<01:01,  6.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  87%|████████▋ | 2611/3000 [06:15<00:46,  8.28it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  87%|████████▋ | 2613/3000 [06:15<00:53,  7.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  87%|████████▋ | 2615/3000 [06:16<00:53,  7.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  87%|████████▋ | 2616/3000 [06:16<01:03,  6.04it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  87%|████████▋ | 2619/3000 [06:16<00:57,  6.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  87%|████████▋ | 2620/3000 [06:16<00:53,  7.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  87%|████████▋ | 2624/3000 [06:17<00:41,  9.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  88%|████████▊ | 2626/3000 [06:17<01:03,  5.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  88%|████████▊ | 2628/3000 [06:18<00:54,  6.77it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  88%|████████▊ | 2629/3000 [06:18<01:02,  5.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  88%|████████▊ | 2631/3000 [06:18<01:06,  5.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  88%|████████▊ | 2634/3000 [06:19<01:00,  6.06it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  88%|████████▊ | 2636/3000 [06:19<00:53,  6.78it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  88%|████████▊ | 2638/3000 [06:19<00:48,  7.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  88%|████████▊ | 2640/3000 [06:19<00:46,  7.74it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  88%|████████▊ | 2641/3000 [06:20<01:02,  5.71it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  88%|████████▊ | 2645/3000 [06:20<00:43,  8.16it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  88%|████████▊ | 2648/3000 [06:21<01:11,  4.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  88%|████████▊ | 2651/3000 [06:21<00:54,  6.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  88%|████████▊ | 2654/3000 [06:22<00:51,  6.78it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  89%|████████▊ | 2658/3000 [06:22<00:44,  7.74it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  89%|████████▊ | 2659/3000 [06:22<00:44,  7.66it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  89%|████████▊ | 2661/3000 [06:23<00:47,  7.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  89%|████████▉ | 2664/3000 [06:23<00:44,  7.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  89%|████████▉ | 2667/3000 [06:23<00:30, 10.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  89%|████████▉ | 2669/3000 [06:24<00:46,  7.06it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  89%|████████▉ | 2673/3000 [06:24<00:41,  7.86it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  89%|████████▉ | 2676/3000 [06:25<00:43,  7.38it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  89%|████████▉ | 2679/3000 [06:25<00:51,  6.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  89%|████████▉ | 2680/3000 [06:26<00:51,  6.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  89%|████████▉ | 2683/3000 [06:26<00:42,  7.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  90%|████████▉ | 2685/3000 [06:26<00:34,  9.14it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  90%|████████▉ | 2687/3000 [06:26<00:33,  9.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  90%|████████▉ | 2689/3000 [06:27<00:42,  7.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  90%|████████▉ | 2691/3000 [06:27<00:47,  6.57it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  90%|████████▉ | 2694/3000 [06:27<00:44,  6.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  90%|████████▉ | 2697/3000 [06:28<00:52,  5.82it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  90%|████████▉ | 2699/3000 [06:28<00:41,  7.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  90%|█████████ | 2704/3000 [06:29<00:28, 10.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  90%|█████████ | 2707/3000 [06:29<00:46,  6.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  90%|█████████ | 2708/3000 [06:30<00:49,  5.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  90%|█████████ | 2711/3000 [06:30<00:45,  6.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  90%|█████████ | 2713/3000 [06:30<00:37,  7.60it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  90%|█████████ | 2714/3000 [06:31<00:55,  5.14it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  91%|█████████ | 2718/3000 [06:31<00:50,  5.63it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  91%|█████████ | 2724/3000 [06:32<00:29,  9.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  91%|█████████ | 2726/3000 [06:32<00:36,  7.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  91%|█████████ | 2728/3000 [06:33<00:52,  5.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  91%|█████████ | 2730/3000 [06:33<00:43,  6.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  91%|█████████ | 2731/3000 [06:33<00:42,  6.40it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  91%|█████████ | 2735/3000 [06:34<00:35,  7.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  91%|█████████ | 2737/3000 [06:34<00:31,  8.36it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  91%|█████████▏| 2740/3000 [06:34<00:35,  7.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  91%|█████████▏| 2742/3000 [06:35<00:30,  8.58it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  92%|█████████▏| 2747/3000 [06:36<00:43,  5.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  92%|█████████▏| 2749/3000 [06:36<00:36,  6.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  92%|█████████▏| 2751/3000 [06:36<00:36,  6.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  92%|█████████▏| 2755/3000 [06:37<00:26,  9.29it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  92%|█████████▏| 2759/3000 [06:38<00:39,  6.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  92%|█████████▏| 2762/3000 [06:38<00:30,  7.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  92%|█████████▏| 2764/3000 [06:38<00:29,  7.93it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  92%|█████████▏| 2766/3000 [06:38<00:28,  8.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  92%|█████████▏| 2770/3000 [06:39<00:33,  6.83it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  92%|█████████▏| 2771/3000 [06:40<00:55,  4.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  92%|█████████▏| 2773/3000 [06:40<00:45,  4.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  93%|█████████▎| 2777/3000 [06:40<00:31,  7.06it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  93%|█████████▎| 2781/3000 [06:41<00:22,  9.77it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  93%|█████████▎| 2783/3000 [06:41<00:34,  6.27it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  93%|█████████▎| 2785/3000 [06:42<00:39,  5.49it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  93%|█████████▎| 2787/3000 [06:42<00:37,  5.67it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  93%|█████████▎| 2792/3000 [06:43<00:30,  6.92it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  93%|█████████▎| 2795/3000 [06:43<00:23,  8.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  93%|█████████▎| 2797/3000 [06:43<00:27,  7.38it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  93%|█████████▎| 2800/3000 [06:44<00:27,  7.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  93%|█████████▎| 2804/3000 [06:44<00:22,  8.77it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  94%|█████████▎| 2808/3000 [06:45<00:27,  7.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  94%|█████████▎| 2810/3000 [06:45<00:26,  7.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  94%|█████████▎| 2812/3000 [06:45<00:27,  6.87it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  94%|█████████▍| 2813/3000 [06:46<00:29,  6.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  94%|█████████▍| 2815/3000 [06:46<00:26,  6.92it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  94%|█████████▍| 2818/3000 [06:46<00:22,  8.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  94%|█████████▍| 2820/3000 [06:46<00:19,  9.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  94%|█████████▍| 2822/3000 [06:47<00:21,  8.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  94%|█████████▍| 2825/3000 [06:47<00:19,  8.83it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  94%|█████████▍| 2826/3000 [06:47<00:20,  8.66it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  94%|█████████▍| 2829/3000 [06:48<00:25,  6.78it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  94%|█████████▍| 2831/3000 [06:48<00:24,  6.88it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  94%|█████████▍| 2833/3000 [06:48<00:22,  7.49it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  94%|█████████▍| 2835/3000 [06:48<00:22,  7.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  95%|█████████▍| 2837/3000 [06:49<00:28,  5.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  95%|█████████▍| 2840/3000 [06:49<00:26,  6.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  95%|█████████▍| 2843/3000 [06:50<00:25,  6.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  95%|█████████▍| 2845/3000 [06:50<00:26,  5.88it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  95%|█████████▍| 2847/3000 [06:51<00:26,  5.83it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  95%|█████████▌| 2850/3000 [06:51<00:22,  6.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  95%|█████████▌| 2854/3000 [06:51<00:14,  9.83it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  95%|█████████▌| 2856/3000 [06:51<00:15,  9.16it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  95%|█████████▌| 2858/3000 [06:52<00:17,  8.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  95%|█████████▌| 2860/3000 [06:52<00:20,  6.74it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  95%|█████████▌| 2862/3000 [06:53<00:26,  5.29it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  95%|█████████▌| 2863/3000 [06:53<00:25,  5.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  96%|█████████▌| 2868/3000 [06:53<00:15,  8.61it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  96%|█████████▌| 2872/3000 [06:54<00:14,  8.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  96%|█████████▌| 2874/3000 [06:54<00:12, 10.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  96%|█████████▌| 2877/3000 [06:55<00:18,  6.55it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  96%|█████████▌| 2879/3000 [06:55<00:15,  7.84it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  96%|█████████▌| 2881/3000 [06:55<00:17,  6.67it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  96%|█████████▌| 2883/3000 [06:56<00:17,  6.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  96%|█████████▌| 2887/3000 [06:56<00:15,  7.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  96%|█████████▋| 2890/3000 [06:56<00:12,  8.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  96%|█████████▋| 2892/3000 [06:57<00:13,  7.77it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  96%|█████████▋| 2893/3000 [06:57<00:17,  6.17it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  96%|█████████▋| 2895/3000 [06:57<00:15,  6.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  97%|█████████▋| 2898/3000 [06:58<00:13,  7.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  97%|█████████▋| 2901/3000 [06:58<00:11,  8.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  97%|█████████▋| 2903/3000 [06:58<00:12,  7.60it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  97%|█████████▋| 2904/3000 [06:59<00:14,  6.66it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  97%|█████████▋| 2907/3000 [06:59<00:11,  8.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  97%|█████████▋| 2910/3000 [06:59<00:12,  7.48it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  97%|█████████▋| 2913/3000 [07:00<00:11,  7.47it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  97%|█████████▋| 2915/3000 [07:00<00:13,  6.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  97%|█████████▋| 2919/3000 [07:01<00:10,  7.87it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  97%|█████████▋| 2921/3000 [07:01<00:10,  7.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  98%|█████████▊| 2925/3000 [07:02<00:16,  4.58it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  98%|█████████▊| 2929/3000 [07:03<00:10,  7.06it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  98%|█████████▊| 2932/3000 [07:03<00:07,  9.55it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  98%|█████████▊| 2934/3000 [07:03<00:08,  8.09it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  98%|█████████▊| 2936/3000 [07:03<00:07,  8.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  98%|█████████▊| 2938/3000 [07:04<00:09,  6.67it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  98%|█████████▊| 2940/3000 [07:04<00:11,  5.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  98%|█████████▊| 2941/3000 [07:05<00:12,  4.67it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  98%|█████████▊| 2945/3000 [07:05<00:07,  6.96it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  98%|█████████▊| 2947/3000 [07:05<00:07,  7.48it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  98%|█████████▊| 2950/3000 [07:06<00:05,  8.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  98%|█████████▊| 2954/3000 [07:06<00:05,  8.92it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  99%|█████████▊| 2956/3000 [07:07<00:07,  5.82it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  99%|█████████▊| 2957/3000 [07:07<00:06,  6.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  99%|█████████▊| 2958/3000 [07:07<00:08,  5.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  99%|█████████▉| 2964/3000 [07:08<00:03,  9.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  99%|█████████▉| 2968/3000 [07:08<00:03,  9.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  99%|█████████▉| 2970/3000 [07:09<00:06,  4.58it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  99%|█████████▉| 2973/3000 [07:09<00:04,  6.11it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  99%|█████████▉| 2976/3000 [07:10<00:03,  6.38it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  99%|█████████▉| 2977/3000 [07:10<00:05,  3.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  99%|█████████▉| 2981/3000 [07:11<00:03,  5.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow: 100%|█████████▉| 2985/3000 [07:11<00:02,  7.06it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow: 100%|█████████▉| 2987/3000 [07:11<00:01,  7.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow: 100%|█████████▉| 2990/3000 [07:12<00:01,  8.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow: 100%|█████████▉| 2993/3000 [07:12<00:00,  7.14it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow: 100%|█████████▉| 2994/3000 [07:13<00:01,  3.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow: 100%|█████████▉| 2997/3000 [07:14<00:00,  3.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow: 100%|█████████▉| 2998/3000 [07:14<00:00,  4.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow: 100%|█████████▉| 2999/3000 [07:15<00:00,  3.02it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow: 100%|██████████| 3000/3000 [07:31<00:00,  6.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "\u001b[32m2026-01-04 10:03:31.374\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36m__main__\u001b[0m:\u001b[36m<module>\u001b[0m:\u001b[36m13\u001b[0m - \u001b[1mEvaluation metrics (after optimization): {'f1': 0.0, 'em': 0.0, 'acc': 0.6646666666666666}\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "logger.info(\"Evaluating workflow on test set...\")\n",
    "with suppress_logger_info():\n",
    "    results = textgrad_optimizer.evaluate(dataset=benchmark, eval_mode=\"test\")\n",
    "logger.info(f\"Evaluation metrics (before optimization): {results}\")\n",
    "\n",
    "logger.info(\"Optimizing workflow...\")\n",
    "textgrad_optimizer.optimize(benchmark, seed=8)\n",
    "textgrad_optimizer.restore_best_graph()\n",
    "\n",
    "logger.info(\"Evaluating workflow on test set...\")\n",
    "with suppress_logger_info():\n",
    "    results = textgrad_optimizer.evaluate(dataset=benchmark, eval_mode=\"test\")\n",
    "logger.info(f\"Evaluation metrics (after optimization): {results}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "4b6f274d",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# with suppress_logger_info():\n",
    "#     results = textgrad_optimizer.evaluate(dataset=benchmark, eval_mode=\"test\")\n",
    "# logger.info(f\"Evaluation metrics (after optimization): {results}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "c5713de9",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2026-01-04 10:03:31.449\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.workflow.workflow_graph\u001b[0m:\u001b[36msave_module\u001b[0m:\u001b[36m1204\u001b[0m - \u001b[1mSaving SequentialWorkFlowGraph to ./debug/textgradinfo_adamson.json\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "textgrad_optimizer.save(\"./debug/textgradinfo_adamson.json\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "16a8bf00",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2026-01-03 19:18:27.139\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.benchmark.pertqa\u001b[0m:\u001b[36m_load_data_from_file\u001b[0m:\u001b[36m52\u001b[0m - \u001b[1mloading HotPotQA data from /home/tl688/pitl688/selfevolve/EvoAgentX/examples/pertqa/norman_train.json ...\u001b[0m\n",
      "\u001b[32m2026-01-03 19:18:27.141\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.benchmark.pertqa\u001b[0m:\u001b[36m_load_data_from_file\u001b[0m:\u001b[36m52\u001b[0m - \u001b[1mloading HotPotQA data from /home/tl688/pitl688/selfevolve/EvoAgentX/examples/pertqa/norman_train.json ...\u001b[0m\n",
      "\u001b[32m2026-01-03 19:18:27.141\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.benchmark.pertqa\u001b[0m:\u001b[36m_load_data_from_file\u001b[0m:\u001b[36m52\u001b[0m - \u001b[1mloading HotPotQA data from /home/tl688/pitl688/selfevolve/EvoAgentX/examples/pertqa/norman_test.json ...\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "benchmark = PertQA(pertdata='norman')\n",
    "workflow_graph = SequentialWorkFlowGraph.from_dict(hotpotqa_graph_data)\n",
    "agent_manager = AgentManager()\n",
    "agent_manager.add_agents_from_workflow(workflow_graph, executor_llm.config)\n",
    "\n",
    "evaluator = Evaluator(\n",
    "    llm=executor_llm, \n",
    "    agent_manager=agent_manager, \n",
    "    collate_func=collate_func, \n",
    "    num_workers=20, \n",
    "    verbose=True\n",
    ")\n",
    "\n",
    "textgrad_optimizer = TextGradOptimizer(\n",
    "    graph=workflow_graph, \n",
    "    optimize_mode=\"all\",\n",
    "    executor_llm=executor_llm, \n",
    "    optimizer_llm=optimizer_llm,\n",
    "    batch_size=3,\n",
    "    max_steps=20,\n",
    "    evaluator=evaluator,\n",
    "    eval_every_n_steps=1,\n",
    "    eval_rounds=1,\n",
    "    save_interval=None,\n",
    "    save_path=\"./\",\n",
    "    rollback=True,\n",
    "    constraints=[]\n",
    ")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "b424bf18",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   0%|          | 2/3000 [00:02<55:54,  1.12s/it]  "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   0%|          | 8/3000 [00:03<12:00,  4.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   0%|          | 12/3000 [00:03<08:17,  6.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   0%|          | 14/3000 [00:04<08:38,  5.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   1%|          | 16/3000 [00:04<09:02,  5.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   1%|          | 18/3000 [00:04<08:21,  5.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   1%|          | 20/3000 [00:04<07:24,  6.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   1%|          | 24/3000 [00:05<05:29,  9.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   1%|          | 28/3000 [00:05<06:26,  7.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   1%|          | 29/3000 [00:06<06:53,  7.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   1%|          | 30/3000 [00:06<07:56,  6.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   1%|          | 32/3000 [00:06<07:51,  6.29it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   1%|          | 33/3000 [00:06<08:17,  5.96it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   1%|          | 34/3000 [00:07<10:44,  4.61it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   1%|          | 36/3000 [00:07<10:29,  4.71it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   1%|▏         | 43/3000 [00:08<05:16,  9.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   2%|▏         | 45/3000 [00:08<07:42,  6.38it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   2%|▏         | 49/3000 [00:08<05:20,  9.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   2%|▏         | 51/3000 [00:09<06:09,  7.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   2%|▏         | 54/3000 [00:10<09:19,  5.27it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   2%|▏         | 57/3000 [00:10<06:43,  7.29it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   2%|▏         | 59/3000 [00:10<06:41,  7.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   2%|▏         | 62/3000 [00:11<10:26,  4.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   2%|▏         | 65/3000 [00:12<07:57,  6.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   2%|▏         | 68/3000 [00:12<05:14,  9.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   2%|▏         | 70/3000 [00:12<06:09,  7.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   2%|▏         | 72/3000 [00:12<05:48,  8.40it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   2%|▏         | 74/3000 [00:13<10:25,  4.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   3%|▎         | 76/3000 [00:13<10:15,  4.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   3%|▎         | 77/3000 [00:14<11:08,  4.37it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   3%|▎         | 80/3000 [00:14<08:39,  5.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   3%|▎         | 81/3000 [00:14<07:52,  6.17it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   3%|▎         | 85/3000 [00:15<05:40,  8.57it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   3%|▎         | 87/3000 [00:15<04:46, 10.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   3%|▎         | 89/3000 [00:15<05:19,  9.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   3%|▎         | 91/3000 [00:16<10:14,  4.74it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   3%|▎         | 95/3000 [00:17<08:33,  5.66it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   3%|▎         | 100/3000 [00:17<07:31,  6.42it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   3%|▎         | 103/3000 [00:18<06:41,  7.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   4%|▎         | 105/3000 [00:18<07:03,  6.84it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   4%|▎         | 107/3000 [00:18<06:28,  7.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   4%|▎         | 108/3000 [00:19<11:41,  4.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   4%|▎         | 110/3000 [00:19<11:42,  4.11it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   4%|▍         | 115/3000 [00:20<06:37,  7.27it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   4%|▍         | 117/3000 [00:20<05:46,  8.32it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   4%|▍         | 122/3000 [00:20<05:11,  9.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   4%|▍         | 125/3000 [00:21<07:01,  6.82it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   4%|▍         | 127/3000 [00:21<06:41,  7.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   4%|▍         | 128/3000 [00:22<09:39,  4.96it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   4%|▍         | 129/3000 [00:22<10:26,  4.58it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   4%|▍         | 131/3000 [00:22<08:14,  5.80it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   4%|▍         | 133/3000 [00:22<07:43,  6.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   4%|▍         | 135/3000 [00:23<07:39,  6.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   5%|▍         | 141/3000 [00:23<05:38,  8.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   5%|▍         | 146/3000 [00:24<05:20,  8.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   5%|▍         | 149/3000 [00:25<08:22,  5.67it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   5%|▌         | 150/3000 [00:25<07:41,  6.17it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   5%|▌         | 153/3000 [00:25<06:58,  6.80it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   5%|▌         | 155/3000 [00:25<05:55,  8.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   5%|▌         | 158/3000 [00:26<06:20,  7.47it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   5%|▌         | 161/3000 [00:26<04:11, 11.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   5%|▌         | 163/3000 [00:26<04:28, 10.58it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   6%|▌         | 166/3000 [00:27<06:53,  6.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   6%|▌         | 169/3000 [00:27<07:49,  6.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   6%|▌         | 177/3000 [00:28<05:21,  8.78it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   6%|▌         | 179/3000 [00:29<05:48,  8.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   6%|▌         | 182/3000 [00:29<04:56,  9.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   6%|▌         | 184/3000 [00:29<05:07,  9.14it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   6%|▌         | 187/3000 [00:30<07:05,  6.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   6%|▋         | 188/3000 [00:30<08:54,  5.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   6%|▋         | 190/3000 [00:31<09:18,  5.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   6%|▋         | 192/3000 [00:31<08:33,  5.46it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   6%|▋         | 194/3000 [00:31<08:24,  5.57it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   7%|▋         | 198/3000 [00:32<06:07,  7.63it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   7%|▋         | 200/3000 [00:32<08:33,  5.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   7%|▋         | 201/3000 [00:33<09:41,  4.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   7%|▋         | 205/3000 [00:33<06:56,  6.71it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   7%|▋         | 208/3000 [00:33<06:43,  6.92it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   7%|▋         | 210/3000 [00:34<08:05,  5.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   7%|▋         | 214/3000 [00:34<05:48,  7.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   7%|▋         | 217/3000 [00:35<07:14,  6.40it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   7%|▋         | 220/3000 [00:35<04:56,  9.38it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   7%|▋         | 222/3000 [00:35<05:17,  8.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   7%|▋         | 224/3000 [00:36<06:00,  7.71it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   8%|▊         | 225/3000 [00:36<07:34,  6.11it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   8%|▊         | 230/3000 [00:37<08:10,  5.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   8%|▊         | 232/3000 [00:37<08:04,  5.71it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   8%|▊         | 235/3000 [00:38<05:49,  7.92it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   8%|▊         | 236/3000 [00:38<06:19,  7.29it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   8%|▊         | 238/3000 [00:38<07:15,  6.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   8%|▊         | 243/3000 [00:39<06:28,  7.09it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   8%|▊         | 244/3000 [00:39<10:23,  4.42it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   8%|▊         | 246/3000 [00:40<09:45,  4.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   8%|▊         | 251/3000 [00:40<05:48,  7.88it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   8%|▊         | 255/3000 [00:41<04:29, 10.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   9%|▊         | 257/3000 [00:41<08:39,  5.28it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   9%|▊         | 259/3000 [00:42<07:41,  5.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   9%|▊         | 261/3000 [00:42<08:20,  5.47it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   9%|▉         | 265/3000 [00:43<07:28,  6.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   9%|▉         | 269/3000 [00:43<06:26,  7.07it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   9%|▉         | 271/3000 [00:43<05:48,  7.83it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   9%|▉         | 274/3000 [00:44<07:11,  6.32it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   9%|▉         | 276/3000 [00:45<09:14,  4.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   9%|▉         | 279/3000 [00:45<08:05,  5.60it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   9%|▉         | 283/3000 [00:46<06:15,  7.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  10%|▉         | 287/3000 [00:46<07:23,  6.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  10%|▉         | 290/3000 [00:47<05:29,  8.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  10%|▉         | 294/3000 [00:47<05:35,  8.05it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  10%|▉         | 299/3000 [00:48<06:29,  6.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  10%|█         | 304/3000 [00:49<05:52,  7.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  10%|█         | 306/3000 [00:49<05:58,  7.52it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  10%|█         | 308/3000 [00:49<05:23,  8.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  10%|█         | 310/3000 [00:50<07:21,  6.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  10%|█         | 312/3000 [00:50<08:16,  5.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  10%|█         | 313/3000 [00:50<09:01,  4.96it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  10%|█         | 315/3000 [00:51<09:40,  4.63it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  11%|█         | 318/3000 [00:51<06:17,  7.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  11%|█         | 320/3000 [00:51<05:28,  8.16it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  11%|█         | 323/3000 [00:52<05:16,  8.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  11%|█         | 325/3000 [00:52<07:35,  5.88it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  11%|█         | 328/3000 [00:53<07:06,  6.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  11%|█         | 331/3000 [00:53<05:54,  7.52it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  11%|█         | 332/3000 [00:53<07:03,  6.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  11%|█         | 337/3000 [00:54<05:46,  7.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  11%|█▏        | 338/3000 [00:54<05:38,  7.86it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  11%|█▏        | 339/3000 [00:54<06:57,  6.38it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  11%|█▏        | 342/3000 [00:55<06:21,  6.97it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  12%|█▏        | 345/3000 [00:55<06:28,  6.83it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  12%|█▏        | 347/3000 [00:56<07:02,  6.28it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  12%|█▏        | 352/3000 [00:56<03:47, 11.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  12%|█▏        | 356/3000 [00:56<05:16,  8.35it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  12%|█▏        | 358/3000 [00:57<05:59,  7.35it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  12%|█▏        | 360/3000 [00:57<05:56,  7.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  12%|█▏        | 362/3000 [00:58<07:29,  5.87it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  12%|█▏        | 364/3000 [00:58<05:41,  7.71it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  12%|█▏        | 368/3000 [00:58<06:24,  6.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  12%|█▏        | 369/3000 [00:58<06:37,  6.61it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  12%|█▏        | 371/3000 [00:59<06:59,  6.27it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  12%|█▎        | 375/3000 [00:59<04:55,  8.87it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  13%|█▎        | 378/3000 [01:00<05:34,  7.83it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  13%|█▎        | 379/3000 [01:00<05:29,  7.96it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  13%|█▎        | 381/3000 [01:00<06:34,  6.63it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  13%|█▎        | 382/3000 [01:00<07:10,  6.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  13%|█▎        | 383/3000 [01:01<08:08,  5.35it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  13%|█▎        | 384/3000 [01:01<09:29,  4.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  13%|█▎        | 386/3000 [01:01<09:02,  4.82it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  13%|█▎        | 392/3000 [01:02<05:27,  7.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  13%|█▎        | 393/3000 [01:02<05:56,  7.32it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  13%|█▎        | 394/3000 [01:02<08:52,  4.89it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  13%|█▎        | 398/3000 [01:03<05:43,  7.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  13%|█▎        | 401/3000 [01:03<04:29,  9.63it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  13%|█▎        | 403/3000 [01:03<04:31,  9.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  14%|█▎        | 405/3000 [01:03<04:41,  9.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  14%|█▎        | 407/3000 [01:04<05:16,  8.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  14%|█▎        | 409/3000 [01:04<06:44,  6.40it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  14%|█▎        | 412/3000 [01:05<06:34,  6.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  14%|█▍        | 415/3000 [01:05<05:38,  7.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  14%|█▍        | 416/3000 [01:05<07:20,  5.87it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  14%|█▍        | 417/3000 [01:06<07:56,  5.42it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  14%|█▍        | 420/3000 [01:06<07:41,  5.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  14%|█▍        | 423/3000 [01:06<05:40,  7.57it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  14%|█▍        | 428/3000 [01:07<04:44,  9.04it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  14%|█▍        | 430/3000 [01:07<05:11,  8.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  14%|█▍        | 432/3000 [01:08<06:19,  6.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  14%|█▍        | 435/3000 [01:08<06:56,  6.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  15%|█▍        | 438/3000 [01:09<07:18,  5.84it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  15%|█▍        | 442/3000 [01:09<05:10,  8.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  15%|█▍        | 444/3000 [01:09<05:42,  7.47it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  15%|█▍        | 448/3000 [01:10<05:21,  7.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  15%|█▌        | 450/3000 [01:10<05:34,  7.63it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  15%|█▌        | 452/3000 [01:11<05:15,  8.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  15%|█▌        | 454/3000 [01:11<05:47,  7.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  15%|█▌        | 457/3000 [01:12<07:23,  5.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  15%|█▌        | 462/3000 [01:12<05:02,  8.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  15%|█▌        | 464/3000 [01:12<04:40,  9.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  16%|█▌        | 466/3000 [01:13<06:07,  6.89it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  16%|█▌        | 471/3000 [01:13<04:31,  9.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  16%|█▌        | 474/3000 [01:13<03:49, 11.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  16%|█▌        | 476/3000 [01:13<03:59, 10.52it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  16%|█▌        | 480/3000 [01:14<05:12,  8.07it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  16%|█▌        | 483/3000 [01:15<07:11,  5.83it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  16%|█▌        | 485/3000 [01:15<06:26,  6.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  16%|█▌        | 486/3000 [01:15<06:49,  6.14it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  16%|█▋        | 488/3000 [01:16<08:06,  5.16it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  16%|█▋        | 490/3000 [01:16<06:07,  6.82it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  16%|█▋        | 493/3000 [01:16<04:39,  8.97it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  16%|█▋        | 495/3000 [01:16<04:33,  9.17it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  17%|█▋        | 498/3000 [01:17<06:06,  6.83it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  17%|█▋        | 499/3000 [01:18<08:45,  4.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  17%|█▋        | 503/3000 [01:18<06:23,  6.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  17%|█▋        | 505/3000 [01:18<05:12,  7.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  17%|█▋        | 507/3000 [01:18<05:13,  7.96it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  17%|█▋        | 509/3000 [01:19<04:49,  8.61it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  17%|█▋        | 511/3000 [01:19<04:37,  8.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  17%|█▋        | 514/3000 [01:20<06:39,  6.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  17%|█▋        | 518/3000 [01:20<04:24,  9.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  17%|█▋        | 520/3000 [01:20<05:22,  7.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  17%|█▋        | 522/3000 [01:21<05:58,  6.92it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  18%|█▊        | 526/3000 [01:21<05:11,  7.93it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  18%|█▊        | 528/3000 [01:22<06:08,  6.71it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  18%|█▊        | 530/3000 [01:22<07:26,  5.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  18%|█▊        | 531/3000 [01:22<08:00,  5.14it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  18%|█▊        | 535/3000 [01:23<04:48,  8.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  18%|█▊        | 537/3000 [01:23<05:48,  7.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  18%|█▊        | 538/3000 [01:23<05:51,  7.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  18%|█▊        | 539/3000 [01:23<08:13,  4.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  18%|█▊        | 541/3000 [01:24<09:02,  4.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  18%|█▊        | 544/3000 [01:24<07:58,  5.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  18%|█▊        | 546/3000 [01:25<06:03,  6.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  18%|█▊        | 550/3000 [01:25<04:26,  9.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  18%|█▊        | 552/3000 [01:25<03:59, 10.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  18%|█▊        | 554/3000 [01:26<07:50,  5.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  19%|█▊        | 556/3000 [01:26<07:49,  5.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  19%|█▊        | 557/3000 [01:27<07:58,  5.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  19%|█▊        | 562/3000 [01:27<05:33,  7.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  19%|█▉        | 564/3000 [01:28<07:23,  5.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  19%|█▉        | 567/3000 [01:28<05:30,  7.35it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  19%|█▉        | 568/3000 [01:28<06:43,  6.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  19%|█▉        | 573/3000 [01:28<04:13,  9.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  19%|█▉        | 575/3000 [01:29<03:54, 10.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  19%|█▉        | 577/3000 [01:29<04:53,  8.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  19%|█▉        | 579/3000 [01:29<05:19,  7.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  19%|█▉        | 581/3000 [01:30<05:41,  7.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  19%|█▉        | 584/3000 [01:30<06:00,  6.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  20%|█▉        | 585/3000 [01:30<07:18,  5.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  20%|█▉        | 589/3000 [01:31<06:10,  6.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  20%|█▉        | 592/3000 [01:31<04:58,  8.05it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  20%|█▉        | 595/3000 [01:32<04:29,  8.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  20%|█▉        | 599/3000 [01:33<06:21,  6.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  20%|██        | 602/3000 [01:33<05:22,  7.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  20%|██        | 606/3000 [01:33<04:42,  8.46it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  20%|██        | 608/3000 [01:34<04:53,  8.16it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  20%|██        | 610/3000 [01:34<06:07,  6.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  20%|██        | 613/3000 [01:35<07:37,  5.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  20%|██        | 615/3000 [01:35<08:16,  4.80it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  21%|██        | 616/3000 [01:36<10:07,  3.93it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  21%|██        | 619/3000 [01:36<06:56,  5.72it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  21%|██        | 623/3000 [01:36<04:38,  8.52it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  21%|██        | 625/3000 [01:37<04:29,  8.82it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  21%|██        | 628/3000 [01:37<05:19,  7.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  21%|██        | 632/3000 [01:37<04:06,  9.60it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  21%|██        | 634/3000 [01:38<06:44,  5.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  21%|██▏       | 638/3000 [01:39<06:05,  6.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  21%|██▏       | 641/3000 [01:39<04:37,  8.49it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  21%|██▏       | 644/3000 [01:40<06:10,  6.36it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  22%|██▏       | 647/3000 [01:40<04:51,  8.06it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  22%|██▏       | 649/3000 [01:40<04:53,  8.02it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  22%|██▏       | 651/3000 [01:41<07:32,  5.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  22%|██▏       | 656/3000 [01:41<05:07,  7.63it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  22%|██▏       | 658/3000 [01:41<04:57,  7.88it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  22%|██▏       | 659/3000 [01:42<04:55,  7.92it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  22%|██▏       | 663/3000 [01:42<04:12,  9.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  22%|██▏       | 665/3000 [01:42<03:44, 10.38it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  22%|██▏       | 668/3000 [01:43<06:09,  6.32it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  22%|██▏       | 670/3000 [01:43<07:34,  5.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  22%|██▏       | 674/3000 [01:44<05:33,  6.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  23%|██▎       | 679/3000 [01:44<03:52,  9.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  23%|██▎       | 681/3000 [01:44<04:09,  9.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  23%|██▎       | 683/3000 [01:45<04:36,  8.38it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  23%|██▎       | 686/3000 [01:45<06:27,  5.97it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  23%|██▎       | 689/3000 [01:46<05:37,  6.84it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  23%|██▎       | 691/3000 [01:46<04:30,  8.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  23%|██▎       | 693/3000 [01:46<04:22,  8.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  23%|██▎       | 695/3000 [01:47<05:14,  7.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  23%|██▎       | 697/3000 [01:47<05:41,  6.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  23%|██▎       | 698/3000 [01:47<07:24,  5.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  23%|██▎       | 700/3000 [01:48<07:53,  4.86it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  24%|██▎       | 705/3000 [01:48<05:13,  7.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  24%|██▎       | 708/3000 [01:49<05:11,  7.36it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  24%|██▎       | 709/3000 [01:49<05:24,  7.06it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  24%|██▎       | 711/3000 [01:49<06:08,  6.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  24%|██▎       | 712/3000 [01:49<07:04,  5.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  24%|██▍       | 715/3000 [01:50<05:22,  7.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  24%|██▍       | 719/3000 [01:50<03:53,  9.78it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  24%|██▍       | 721/3000 [01:50<03:56,  9.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  24%|██▍       | 724/3000 [01:51<07:08,  5.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  24%|██▍       | 728/3000 [01:52<05:15,  7.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  24%|██▍       | 731/3000 [01:52<04:55,  7.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  24%|██▍       | 732/3000 [01:52<06:08,  6.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  24%|██▍       | 735/3000 [01:53<05:28,  6.89it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  25%|██▍       | 737/3000 [01:53<04:54,  7.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  25%|██▍       | 738/3000 [01:53<06:51,  5.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  25%|██▍       | 742/3000 [01:54<04:55,  7.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  25%|██▍       | 743/3000 [01:54<05:40,  6.63it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  25%|██▍       | 746/3000 [01:55<06:04,  6.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  25%|██▍       | 748/3000 [01:55<05:00,  7.49it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  25%|██▌       | 751/3000 [01:55<04:18,  8.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  25%|██▌       | 753/3000 [01:55<04:17,  8.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  25%|██▌       | 757/3000 [01:56<04:38,  8.05it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  25%|██▌       | 759/3000 [01:56<04:23,  8.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  25%|██▌       | 760/3000 [01:56<06:26,  5.80it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  25%|██▌       | 762/3000 [01:57<08:40,  4.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  26%|██▌       | 765/3000 [01:57<05:14,  7.11it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  26%|██▌       | 769/3000 [01:58<04:59,  7.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  26%|██▌       | 774/3000 [01:58<04:08,  8.96it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  26%|██▌       | 777/3000 [01:59<05:58,  6.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  26%|██▌       | 780/3000 [01:59<04:20,  8.52it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  26%|██▌       | 782/3000 [02:00<03:51,  9.58it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  26%|██▌       | 784/3000 [02:00<04:38,  7.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  26%|██▋       | 788/3000 [02:00<03:48,  9.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  26%|██▋       | 790/3000 [02:00<04:06,  8.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  26%|██▋       | 792/3000 [02:01<05:01,  7.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  26%|██▋       | 795/3000 [02:02<07:00,  5.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  27%|██▋       | 797/3000 [02:02<05:45,  6.37it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  27%|██▋       | 799/3000 [02:02<05:25,  6.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  27%|██▋       | 802/3000 [02:03<05:10,  7.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  27%|██▋       | 805/3000 [02:03<04:32,  8.05it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  27%|██▋       | 807/3000 [02:03<03:34, 10.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  27%|██▋       | 809/3000 [02:04<05:46,  6.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  27%|██▋       | 812/3000 [02:04<04:47,  7.61it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  27%|██▋       | 814/3000 [02:04<05:12,  6.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  27%|██▋       | 816/3000 [02:04<04:58,  7.32it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  27%|██▋       | 818/3000 [02:05<04:51,  7.47it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  27%|██▋       | 819/3000 [02:05<04:54,  7.42it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  27%|██▋       | 820/3000 [02:05<05:57,  6.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  27%|██▋       | 821/3000 [02:05<06:18,  5.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  28%|██▊       | 825/3000 [02:06<04:32,  7.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  28%|██▊       | 827/3000 [02:06<03:45,  9.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  28%|██▊       | 830/3000 [02:06<04:16,  8.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  28%|██▊       | 832/3000 [02:07<04:46,  7.57it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  28%|██▊       | 833/3000 [02:07<05:12,  6.93it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  28%|██▊       | 834/3000 [02:07<07:10,  5.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  28%|██▊       | 836/3000 [02:07<06:19,  5.71it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  28%|██▊       | 839/3000 [02:08<05:34,  6.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  28%|██▊       | 840/3000 [02:08<06:04,  5.93it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  28%|██▊       | 843/3000 [02:08<04:51,  7.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  28%|██▊       | 844/3000 [02:09<04:46,  7.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  28%|██▊       | 848/3000 [02:09<03:49,  9.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  28%|██▊       | 850/3000 [02:09<04:56,  7.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  28%|██▊       | 852/3000 [02:10<05:39,  6.32it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  28%|██▊       | 855/3000 [02:10<04:49,  7.42it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  29%|██▊       | 858/3000 [02:11<05:10,  6.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  29%|██▊       | 860/3000 [02:11<04:56,  7.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  29%|██▉       | 863/3000 [02:11<05:21,  6.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  29%|██▉       | 865/3000 [02:12<05:52,  6.06it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  29%|██▉       | 866/3000 [02:12<08:30,  4.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  29%|██▉       | 869/3000 [02:13<06:18,  5.63it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  29%|██▉       | 871/3000 [02:13<06:22,  5.57it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  29%|██▉       | 875/3000 [02:13<03:40,  9.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  29%|██▉       | 877/3000 [02:13<03:16, 10.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  29%|██▉       | 879/3000 [02:14<04:12,  8.40it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  29%|██▉       | 881/3000 [02:14<05:38,  6.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  29%|██▉       | 882/3000 [02:14<05:57,  5.93it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  29%|██▉       | 884/3000 [02:15<05:47,  6.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  30%|██▉       | 886/3000 [02:15<04:22,  8.04it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  30%|██▉       | 887/3000 [02:15<06:18,  5.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  30%|██▉       | 893/3000 [02:16<03:33,  9.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  30%|██▉       | 897/3000 [02:16<04:25,  7.93it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  30%|██▉       | 899/3000 [02:16<03:46,  9.27it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  30%|███       | 903/3000 [02:17<04:09,  8.42it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  30%|███       | 905/3000 [02:18<05:34,  6.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  30%|███       | 907/3000 [02:18<05:38,  6.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  30%|███       | 909/3000 [02:18<05:33,  6.27it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  30%|███       | 911/3000 [02:19<06:28,  5.38it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  31%|███       | 917/3000 [02:19<03:41,  9.42it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  31%|███       | 919/3000 [02:19<03:19, 10.42it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  31%|███       | 921/3000 [02:20<07:03,  4.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  31%|███       | 923/3000 [02:20<05:38,  6.14it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  31%|███       | 925/3000 [02:21<06:25,  5.38it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  31%|███       | 927/3000 [02:21<05:44,  6.02it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  31%|███       | 930/3000 [02:21<05:22,  6.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  31%|███       | 934/3000 [02:22<03:30,  9.80it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  31%|███       | 937/3000 [02:22<05:14,  6.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  31%|███▏      | 939/3000 [02:23<05:51,  5.87it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  31%|███▏      | 941/3000 [02:23<05:30,  6.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  31%|███▏      | 943/3000 [02:23<05:02,  6.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  32%|███▏      | 946/3000 [02:24<03:47,  9.04it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  32%|███▏      | 947/3000 [02:24<05:29,  6.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  32%|███▏      | 949/3000 [02:25<07:54,  4.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  32%|███▏      | 953/3000 [02:25<05:21,  6.37it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  32%|███▏      | 955/3000 [02:25<04:27,  7.63it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  32%|███▏      | 957/3000 [02:25<03:48,  8.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  32%|███▏      | 959/3000 [02:26<03:42,  9.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  32%|███▏      | 962/3000 [02:26<05:21,  6.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  32%|███▏      | 963/3000 [02:27<05:52,  5.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  32%|███▏      | 964/3000 [02:27<06:21,  5.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  32%|███▏      | 971/3000 [02:28<04:30,  7.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  32%|███▏      | 973/3000 [02:28<05:57,  5.67it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  33%|███▎      | 976/3000 [02:29<04:40,  7.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  33%|███▎      | 978/3000 [02:29<04:36,  7.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  33%|███▎      | 979/3000 [02:29<04:59,  6.74it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  33%|███▎      | 983/3000 [02:29<03:35,  9.36it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  33%|███▎      | 987/3000 [02:30<05:08,  6.52it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  33%|███▎      | 989/3000 [02:31<05:02,  6.66it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  33%|███▎      | 991/3000 [02:31<07:41,  4.36it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  33%|███▎      | 995/3000 [02:32<05:16,  6.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  33%|███▎      | 997/3000 [02:33<07:29,  4.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  33%|███▎      | 1002/3000 [02:33<04:45,  7.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  34%|███▎      | 1006/3000 [02:34<04:39,  7.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  34%|███▎      | 1008/3000 [02:34<04:49,  6.88it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  34%|███▎      | 1012/3000 [02:35<04:56,  6.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  34%|███▍      | 1014/3000 [02:35<05:25,  6.09it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  34%|███▍      | 1018/3000 [02:36<04:21,  7.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  34%|███▍      | 1020/3000 [02:36<05:48,  5.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  34%|███▍      | 1022/3000 [02:37<06:31,  5.05it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  34%|███▍      | 1025/3000 [02:37<06:22,  5.17it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  34%|███▍      | 1028/3000 [02:37<04:37,  7.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  34%|███▍      | 1030/3000 [02:38<03:36,  9.09it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  34%|███▍      | 1032/3000 [02:38<05:56,  5.52it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  34%|███▍      | 1033/3000 [02:39<06:55,  4.74it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  34%|███▍      | 1034/3000 [02:39<07:54,  4.14it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  35%|███▍      | 1036/3000 [02:39<06:53,  4.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  35%|███▍      | 1039/3000 [02:40<05:44,  5.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  35%|███▍      | 1042/3000 [02:40<04:23,  7.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  35%|███▍      | 1045/3000 [02:41<05:36,  5.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  35%|███▍      | 1047/3000 [02:41<04:53,  6.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  35%|███▍      | 1048/3000 [02:41<08:27,  3.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  35%|███▌      | 1051/3000 [02:42<06:15,  5.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  35%|███▌      | 1053/3000 [02:42<05:17,  6.14it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  35%|███▌      | 1054/3000 [02:42<07:06,  4.57it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  35%|███▌      | 1056/3000 [02:43<05:34,  5.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  35%|███▌      | 1058/3000 [02:43<05:15,  6.16it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  35%|███▌      | 1059/3000 [02:43<06:46,  4.77it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  35%|███▌      | 1064/3000 [02:44<03:57,  8.17it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  36%|███▌      | 1066/3000 [02:45<06:36,  4.87it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  36%|███▌      | 1067/3000 [02:45<07:42,  4.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  36%|███▌      | 1069/3000 [02:45<07:19,  4.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  36%|███▌      | 1072/3000 [02:46<05:23,  5.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  36%|███▌      | 1073/3000 [02:46<05:37,  5.71it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  36%|███▌      | 1077/3000 [02:46<03:58,  8.06it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  36%|███▌      | 1079/3000 [02:46<04:10,  7.66it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  36%|███▌      | 1082/3000 [02:47<05:23,  5.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  36%|███▌      | 1083/3000 [02:47<05:55,  5.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  36%|███▌      | 1086/3000 [02:48<05:21,  5.96it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  36%|███▌      | 1087/3000 [02:48<04:53,  6.52it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  36%|███▋      | 1089/3000 [02:49<05:54,  5.38it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  36%|███▋      | 1091/3000 [02:49<04:23,  7.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  36%|███▋      | 1094/3000 [02:50<07:26,  4.27it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  37%|███▋      | 1096/3000 [02:50<05:28,  5.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  37%|███▋      | 1098/3000 [02:50<05:46,  5.49it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  37%|███▋      | 1100/3000 [02:51<04:59,  6.35it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  37%|███▋      | 1104/3000 [02:51<04:03,  7.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  37%|███▋      | 1107/3000 [02:51<04:31,  6.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  37%|███▋      | 1109/3000 [02:52<04:54,  6.42it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  37%|███▋      | 1110/3000 [02:53<08:46,  3.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  37%|███▋      | 1111/3000 [02:53<08:23,  3.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  37%|███▋      | 1114/3000 [02:53<06:22,  4.93it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  37%|███▋      | 1116/3000 [02:54<05:36,  5.60it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  37%|███▋      | 1117/3000 [02:54<05:58,  5.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  37%|███▋      | 1119/3000 [02:54<05:04,  6.17it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  37%|███▋      | 1120/3000 [02:54<05:31,  5.67it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  37%|███▋      | 1123/3000 [02:55<04:11,  7.46it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  38%|███▊      | 1126/3000 [02:55<03:48,  8.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  38%|███▊      | 1128/3000 [02:55<05:24,  5.78it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  38%|███▊      | 1133/3000 [02:56<04:12,  7.40it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  38%|███▊      | 1135/3000 [02:56<04:33,  6.83it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  38%|███▊      | 1138/3000 [02:57<03:17,  9.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  38%|███▊      | 1140/3000 [02:57<03:24,  9.09it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  38%|███▊      | 1142/3000 [02:58<06:07,  5.05it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  38%|███▊      | 1144/3000 [02:58<07:29,  4.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  38%|███▊      | 1146/3000 [02:59<05:57,  5.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  38%|███▊      | 1149/3000 [02:59<04:27,  6.93it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  38%|███▊      | 1152/3000 [02:59<03:05,  9.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  38%|███▊      | 1154/3000 [03:00<05:48,  5.29it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  39%|███▊      | 1156/3000 [03:00<06:17,  4.88it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  39%|███▊      | 1158/3000 [03:01<05:14,  5.86it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  39%|███▊      | 1162/3000 [03:01<03:41,  8.28it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  39%|███▉      | 1163/3000 [03:01<04:50,  6.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  39%|███▉      | 1166/3000 [03:02<03:54,  7.82it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  39%|███▉      | 1168/3000 [03:02<03:35,  8.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  39%|███▉      | 1170/3000 [03:02<04:32,  6.71it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  39%|███▉      | 1172/3000 [03:03<04:45,  6.40it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  39%|███▉      | 1174/3000 [03:03<04:59,  6.09it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  39%|███▉      | 1177/3000 [03:03<04:11,  7.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  39%|███▉      | 1180/3000 [03:04<04:03,  7.47it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  39%|███▉      | 1181/3000 [03:04<04:17,  7.06it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  39%|███▉      | 1184/3000 [03:04<03:10,  9.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  40%|███▉      | 1185/3000 [03:04<04:11,  7.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  40%|███▉      | 1190/3000 [03:05<02:54, 10.36it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  40%|███▉      | 1194/3000 [03:05<02:39, 11.29it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  40%|███▉      | 1196/3000 [03:06<07:02,  4.27it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  40%|███▉      | 1197/3000 [03:06<06:38,  4.52it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  40%|████      | 1201/3000 [03:07<04:42,  6.37it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  40%|████      | 1204/3000 [03:07<03:39,  8.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  40%|████      | 1206/3000 [03:07<03:26,  8.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  40%|████      | 1208/3000 [03:07<03:20,  8.92it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  40%|████      | 1210/3000 [03:08<03:57,  7.55it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  40%|████      | 1211/3000 [03:08<03:55,  7.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  40%|████      | 1215/3000 [03:09<05:24,  5.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  41%|████      | 1219/3000 [03:09<04:45,  6.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  41%|████      | 1220/3000 [03:10<04:46,  6.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  41%|████      | 1222/3000 [03:10<04:17,  6.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  41%|████      | 1225/3000 [03:10<03:54,  7.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  41%|████      | 1228/3000 [03:11<03:44,  7.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  41%|████      | 1230/3000 [03:11<04:18,  6.84it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  41%|████      | 1232/3000 [03:11<05:15,  5.61it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  41%|████      | 1235/3000 [03:12<04:45,  6.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  41%|████▏     | 1239/3000 [03:12<03:55,  7.47it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  41%|████▏     | 1242/3000 [03:13<03:30,  8.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  41%|████▏     | 1244/3000 [03:13<03:15,  8.96it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  42%|████▏     | 1247/3000 [03:13<04:04,  7.17it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  42%|████▏     | 1249/3000 [03:14<03:34,  8.16it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  42%|████▏     | 1251/3000 [03:14<03:37,  8.04it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  42%|████▏     | 1252/3000 [03:14<03:51,  7.55it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  42%|████▏     | 1254/3000 [03:14<04:27,  6.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  42%|████▏     | 1256/3000 [03:15<04:31,  6.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  42%|████▏     | 1259/3000 [03:15<03:23,  8.58it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  42%|████▏     | 1261/3000 [03:15<04:29,  6.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  42%|████▏     | 1263/3000 [03:16<06:17,  4.60it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  42%|████▏     | 1266/3000 [03:16<04:09,  6.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  42%|████▏     | 1268/3000 [03:17<05:31,  5.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  42%|████▏     | 1271/3000 [03:17<04:47,  6.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  42%|████▏     | 1274/3000 [03:18<03:36,  7.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  43%|████▎     | 1277/3000 [03:18<04:49,  5.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  43%|████▎     | 1280/3000 [03:19<04:01,  7.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  43%|████▎     | 1281/3000 [03:19<06:49,  4.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  43%|████▎     | 1283/3000 [03:20<06:01,  4.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  43%|████▎     | 1284/3000 [03:20<05:28,  5.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  43%|████▎     | 1285/3000 [03:20<05:46,  4.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  43%|████▎     | 1288/3000 [03:20<04:27,  6.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  43%|████▎     | 1291/3000 [03:21<03:26,  8.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  43%|████▎     | 1294/3000 [03:21<03:42,  7.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  43%|████▎     | 1296/3000 [03:21<04:01,  7.04it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  43%|████▎     | 1298/3000 [03:22<04:27,  6.37it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  43%|████▎     | 1299/3000 [03:22<08:21,  3.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  43%|████▎     | 1302/3000 [03:23<05:54,  4.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  44%|████▎     | 1307/3000 [03:23<04:09,  6.78it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  44%|████▎     | 1312/3000 [03:24<02:59,  9.38it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  44%|████▍     | 1314/3000 [03:24<03:06,  9.05it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  44%|████▍     | 1315/3000 [03:24<04:03,  6.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  44%|████▍     | 1318/3000 [03:25<03:51,  7.27it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  44%|████▍     | 1320/3000 [03:25<03:26,  8.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  44%|████▍     | 1321/3000 [03:25<05:02,  5.55it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  44%|████▍     | 1324/3000 [03:26<05:18,  5.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  44%|████▍     | 1327/3000 [03:26<03:52,  7.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  44%|████▍     | 1328/3000 [03:26<04:35,  6.06it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  44%|████▍     | 1330/3000 [03:27<04:57,  5.61it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  44%|████▍     | 1331/3000 [03:27<05:08,  5.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  44%|████▍     | 1335/3000 [03:27<03:46,  7.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  45%|████▍     | 1338/3000 [03:28<02:53,  9.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  45%|████▍     | 1340/3000 [03:28<03:09,  8.78it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  45%|████▍     | 1342/3000 [03:28<03:53,  7.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  45%|████▍     | 1344/3000 [03:29<05:16,  5.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  45%|████▍     | 1345/3000 [03:29<05:20,  5.17it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  45%|████▍     | 1349/3000 [03:30<03:35,  7.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  45%|████▌     | 1352/3000 [03:30<03:55,  7.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  45%|████▌     | 1355/3000 [03:30<03:39,  7.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  45%|████▌     | 1357/3000 [03:31<04:35,  5.97it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  45%|████▌     | 1358/3000 [03:31<04:48,  5.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  45%|████▌     | 1362/3000 [03:31<03:19,  8.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  45%|████▌     | 1364/3000 [03:32<03:00,  9.07it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  46%|████▌     | 1366/3000 [03:32<04:11,  6.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  46%|████▌     | 1368/3000 [03:32<03:27,  7.87it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  46%|████▌     | 1372/3000 [03:33<03:37,  7.48it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  46%|████▌     | 1373/3000 [03:34<06:09,  4.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  46%|████▌     | 1375/3000 [03:34<05:18,  5.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  46%|████▌     | 1378/3000 [03:34<04:16,  6.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  46%|████▌     | 1380/3000 [03:34<03:32,  7.63it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  46%|████▌     | 1384/3000 [03:35<03:43,  7.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  46%|████▌     | 1387/3000 [03:35<03:36,  7.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  46%|████▋     | 1389/3000 [03:36<04:10,  6.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  46%|████▋     | 1390/3000 [03:36<04:10,  6.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  46%|████▋     | 1391/3000 [03:36<05:17,  5.07it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  46%|████▋     | 1394/3000 [03:37<04:36,  5.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  47%|████▋     | 1398/3000 [03:37<02:56,  9.06it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  47%|████▋     | 1400/3000 [03:37<03:12,  8.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  47%|████▋     | 1401/3000 [03:38<04:37,  5.77it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  47%|████▋     | 1402/3000 [03:38<06:35,  4.04it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  47%|████▋     | 1404/3000 [03:39<06:17,  4.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  47%|████▋     | 1408/3000 [03:39<04:22,  6.06it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  47%|████▋     | 1409/3000 [03:39<04:28,  5.93it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  47%|████▋     | 1413/3000 [03:40<03:48,  6.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  47%|████▋     | 1415/3000 [03:40<03:32,  7.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  47%|████▋     | 1417/3000 [03:40<02:57,  8.89it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  47%|████▋     | 1419/3000 [03:40<02:42,  9.72it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  47%|████▋     | 1421/3000 [03:41<03:50,  6.84it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  47%|████▋     | 1423/3000 [03:41<03:42,  7.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  47%|████▋     | 1424/3000 [03:42<04:54,  5.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  48%|████▊     | 1428/3000 [03:42<03:48,  6.89it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  48%|████▊     | 1430/3000 [03:42<03:36,  7.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  48%|████▊     | 1432/3000 [03:43<04:58,  5.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  48%|████▊     | 1434/3000 [03:43<05:33,  4.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  48%|████▊     | 1436/3000 [03:44<04:41,  5.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  48%|████▊     | 1439/3000 [03:44<03:30,  7.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  48%|████▊     | 1440/3000 [03:44<04:09,  6.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  48%|████▊     | 1443/3000 [03:44<03:25,  7.58it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  48%|████▊     | 1445/3000 [03:45<04:07,  6.28it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  48%|████▊     | 1446/3000 [03:45<05:21,  4.84it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  48%|████▊     | 1452/3000 [03:46<04:12,  6.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  48%|████▊     | 1455/3000 [03:47<04:00,  6.42it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  49%|████▊     | 1459/3000 [03:47<02:48,  9.17it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  49%|████▊     | 1462/3000 [03:47<02:36,  9.84it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  49%|████▉     | 1464/3000 [03:48<02:41,  9.52it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  49%|████▉     | 1466/3000 [03:49<05:21,  4.77it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  49%|████▉     | 1472/3000 [03:49<04:05,  6.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  49%|████▉     | 1476/3000 [03:50<03:46,  6.72it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  49%|████▉     | 1479/3000 [03:50<03:07,  8.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  49%|████▉     | 1481/3000 [03:51<03:50,  6.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  49%|████▉     | 1482/3000 [03:51<04:39,  5.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  49%|████▉     | 1483/3000 [03:51<05:16,  4.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  50%|████▉     | 1486/3000 [03:52<03:56,  6.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  50%|████▉     | 1488/3000 [03:52<04:26,  5.67it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  50%|████▉     | 1492/3000 [03:52<02:57,  8.49it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  50%|████▉     | 1494/3000 [03:53<03:38,  6.89it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  50%|████▉     | 1497/3000 [03:53<03:05,  8.11it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  50%|████▉     | 1499/3000 [03:53<03:47,  6.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  50%|█████     | 1500/3000 [03:54<04:31,  5.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  50%|█████     | 1503/3000 [03:54<04:22,  5.71it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  50%|█████     | 1504/3000 [03:55<05:22,  4.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  50%|█████     | 1509/3000 [03:55<03:12,  7.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  50%|█████     | 1511/3000 [03:56<04:37,  5.37it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  50%|█████     | 1515/3000 [03:56<03:52,  6.38it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  51%|█████     | 1517/3000 [03:57<03:54,  6.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  51%|█████     | 1520/3000 [03:57<03:51,  6.40it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  51%|█████     | 1522/3000 [03:58<04:07,  5.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  51%|█████     | 1523/3000 [03:58<04:10,  5.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  51%|█████     | 1524/3000 [03:58<04:46,  5.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  51%|█████     | 1525/3000 [03:58<05:16,  4.67it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  51%|█████     | 1528/3000 [03:59<03:42,  6.61it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  51%|█████     | 1531/3000 [03:59<03:24,  7.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  51%|█████     | 1532/3000 [03:59<03:33,  6.87it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  51%|█████     | 1535/3000 [04:00<03:52,  6.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  51%|█████▏    | 1539/3000 [04:00<03:05,  7.87it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  51%|█████▏    | 1541/3000 [04:00<02:44,  8.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  51%|█████▏    | 1544/3000 [04:01<03:40,  6.60it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  52%|█████▏    | 1545/3000 [04:01<03:59,  6.07it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  52%|█████▏    | 1547/3000 [04:01<04:13,  5.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  52%|█████▏    | 1551/3000 [04:02<03:28,  6.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  52%|█████▏    | 1552/3000 [04:02<04:16,  5.66it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  52%|█████▏    | 1555/3000 [04:03<03:40,  6.55it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  52%|█████▏    | 1558/3000 [04:03<03:23,  7.09it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  52%|█████▏    | 1560/3000 [04:04<04:22,  5.49it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  52%|█████▏    | 1562/3000 [04:04<04:10,  5.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  52%|█████▏    | 1563/3000 [04:04<04:36,  5.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  52%|█████▏    | 1567/3000 [04:05<03:25,  6.97it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  52%|█████▏    | 1570/3000 [04:05<03:37,  6.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  52%|█████▏    | 1572/3000 [04:05<02:52,  8.27it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  52%|█████▏    | 1574/3000 [04:06<02:34,  9.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  53%|█████▎    | 1576/3000 [04:06<02:36,  9.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  53%|█████▎    | 1578/3000 [04:06<03:37,  6.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  53%|█████▎    | 1580/3000 [04:07<03:37,  6.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  53%|█████▎    | 1581/3000 [04:07<04:01,  5.88it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  53%|█████▎    | 1582/3000 [04:07<06:21,  3.72it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  53%|█████▎    | 1586/3000 [04:08<03:39,  6.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  53%|█████▎    | 1588/3000 [04:08<03:48,  6.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  53%|█████▎    | 1594/3000 [04:09<02:34,  9.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  53%|█████▎    | 1596/3000 [04:09<04:11,  5.58it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  53%|█████▎    | 1598/3000 [04:10<04:13,  5.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  53%|█████▎    | 1600/3000 [04:10<03:53,  6.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  53%|█████▎    | 1601/3000 [04:10<03:51,  6.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  54%|█████▎    | 1606/3000 [04:11<02:20,  9.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  54%|█████▎    | 1609/3000 [04:11<02:00, 11.58it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  54%|█████▎    | 1611/3000 [04:11<02:43,  8.48it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  54%|█████▍    | 1613/3000 [04:11<02:37,  8.82it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  54%|█████▍    | 1615/3000 [04:12<03:46,  6.11it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  54%|█████▍    | 1617/3000 [04:12<03:32,  6.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  54%|█████▍    | 1618/3000 [04:13<04:13,  5.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  54%|█████▍    | 1619/3000 [04:13<06:47,  3.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  54%|█████▍    | 1621/3000 [04:14<07:43,  2.97it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  54%|█████▍    | 1629/3000 [04:15<02:36,  8.74it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  54%|█████▍    | 1631/3000 [04:15<02:28,  9.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  54%|█████▍    | 1633/3000 [04:15<03:11,  7.16it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  55%|█████▍    | 1635/3000 [04:16<04:11,  5.42it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  55%|█████▍    | 1638/3000 [04:16<03:41,  6.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  55%|█████▍    | 1641/3000 [04:17<03:19,  6.83it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  55%|█████▍    | 1645/3000 [04:17<02:29,  9.05it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  55%|█████▍    | 1647/3000 [04:17<02:29,  9.02it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  55%|█████▍    | 1649/3000 [04:18<03:34,  6.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  55%|█████▌    | 1650/3000 [04:18<03:26,  6.55it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  55%|█████▌    | 1653/3000 [04:19<03:57,  5.67it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  55%|█████▌    | 1655/3000 [04:19<04:04,  5.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  55%|█████▌    | 1660/3000 [04:19<02:23,  9.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  55%|█████▌    | 1662/3000 [04:20<03:30,  6.36it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  56%|█████▌    | 1665/3000 [04:20<03:27,  6.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  56%|█████▌    | 1667/3000 [04:21<03:47,  5.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  56%|█████▌    | 1669/3000 [04:21<04:51,  4.57it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  56%|█████▌    | 1673/3000 [04:22<03:06,  7.11it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  56%|█████▌    | 1675/3000 [04:22<02:51,  7.71it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  56%|█████▌    | 1677/3000 [04:22<03:13,  6.82it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  56%|█████▌    | 1679/3000 [04:23<03:10,  6.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  56%|█████▌    | 1680/3000 [04:23<03:25,  6.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  56%|█████▌    | 1682/3000 [04:23<03:11,  6.88it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  56%|█████▌    | 1685/3000 [04:24<03:24,  6.42it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  56%|█████▌    | 1686/3000 [04:24<03:12,  6.84it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  56%|█████▋    | 1688/3000 [04:24<03:26,  6.36it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  56%|█████▋    | 1690/3000 [04:24<03:48,  5.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  56%|█████▋    | 1691/3000 [04:25<03:51,  5.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  56%|█████▋    | 1693/3000 [04:25<05:07,  4.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  57%|█████▋    | 1697/3000 [04:26<03:06,  6.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  57%|█████▋    | 1701/3000 [04:26<01:59, 10.89it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  57%|█████▋    | 1704/3000 [04:26<01:33, 13.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  57%|█████▋    | 1707/3000 [04:27<02:35,  8.29it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  57%|█████▋    | 1709/3000 [04:27<04:19,  4.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  57%|█████▋    | 1711/3000 [04:28<04:28,  4.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  57%|█████▋    | 1714/3000 [04:28<03:36,  5.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  57%|█████▋    | 1716/3000 [04:28<03:14,  6.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  57%|█████▋    | 1717/3000 [04:29<03:32,  6.05it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  57%|█████▋    | 1722/3000 [04:29<02:23,  8.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  57%|█████▋    | 1724/3000 [04:29<02:21,  9.02it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  58%|█████▊    | 1726/3000 [04:30<02:41,  7.89it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  58%|█████▊    | 1727/3000 [04:30<03:36,  5.87it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  58%|█████▊    | 1728/3000 [04:30<03:53,  5.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  58%|█████▊    | 1730/3000 [04:30<03:22,  6.28it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  58%|█████▊    | 1732/3000 [04:31<03:48,  5.55it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  58%|█████▊    | 1734/3000 [04:31<03:14,  6.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  58%|█████▊    | 1737/3000 [04:32<02:46,  7.57it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  58%|█████▊    | 1739/3000 [04:32<02:37,  8.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  58%|█████▊    | 1741/3000 [04:32<02:20,  8.96it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  58%|█████▊    | 1744/3000 [04:33<03:10,  6.58it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  58%|█████▊    | 1747/3000 [04:33<04:00,  5.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  58%|█████▊    | 1750/3000 [04:34<03:11,  6.52it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  58%|█████▊    | 1753/3000 [04:34<03:23,  6.14it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  59%|█████▊    | 1756/3000 [04:34<02:38,  7.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  59%|█████▊    | 1759/3000 [04:35<03:28,  5.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  59%|█████▊    | 1760/3000 [04:35<03:20,  6.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  59%|█████▉    | 1763/3000 [04:36<02:38,  7.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  59%|█████▉    | 1765/3000 [04:36<03:56,  5.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  59%|█████▉    | 1768/3000 [04:37<03:04,  6.66it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  59%|█████▉    | 1770/3000 [04:37<02:50,  7.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  59%|█████▉    | 1773/3000 [04:37<02:08,  9.57it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  59%|█████▉    | 1775/3000 [04:37<02:43,  7.49it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  59%|█████▉    | 1776/3000 [04:38<03:26,  5.93it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  59%|█████▉    | 1778/3000 [04:38<04:11,  4.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  59%|█████▉    | 1781/3000 [04:39<03:07,  6.49it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  60%|█████▉    | 1785/3000 [04:39<03:10,  6.36it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  60%|█████▉    | 1786/3000 [04:40<03:26,  5.87it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  60%|█████▉    | 1789/3000 [04:40<03:17,  6.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  60%|█████▉    | 1791/3000 [04:40<02:43,  7.38it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  60%|█████▉    | 1792/3000 [04:40<03:12,  6.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  60%|█████▉    | 1794/3000 [04:41<03:30,  5.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  60%|█████▉    | 1798/3000 [04:41<03:03,  6.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  60%|█████▉    | 1799/3000 [04:42<03:36,  5.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  60%|██████    | 1802/3000 [04:42<02:58,  6.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  60%|██████    | 1805/3000 [04:43<02:51,  6.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  60%|██████    | 1808/3000 [04:43<02:56,  6.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  60%|██████    | 1811/3000 [04:43<02:15,  8.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  60%|██████    | 1814/3000 [04:43<01:43, 11.49it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  61%|██████    | 1816/3000 [04:44<02:30,  7.88it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  61%|██████    | 1818/3000 [04:45<03:53,  5.07it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  61%|██████    | 1821/3000 [04:45<03:29,  5.63it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  61%|██████    | 1822/3000 [04:45<04:12,  4.67it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  61%|██████    | 1823/3000 [04:46<04:45,  4.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  61%|██████    | 1826/3000 [04:46<03:16,  5.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  61%|██████    | 1828/3000 [04:46<03:04,  6.37it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  61%|██████    | 1830/3000 [04:47<04:02,  4.83it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  61%|██████    | 1832/3000 [04:47<03:37,  5.37it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  61%|██████    | 1834/3000 [04:47<02:57,  6.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  61%|██████    | 1836/3000 [04:48<03:23,  5.72it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  61%|██████▏   | 1838/3000 [04:48<03:40,  5.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  61%|██████▏   | 1839/3000 [04:49<03:44,  5.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  61%|██████▏   | 1840/3000 [04:49<03:54,  4.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  61%|██████▏   | 1842/3000 [04:49<03:41,  5.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  61%|██████▏   | 1844/3000 [04:49<02:29,  7.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  62%|██████▏   | 1845/3000 [04:50<03:16,  5.86it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  62%|██████▏   | 1847/3000 [04:50<03:34,  5.37it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  62%|██████▏   | 1849/3000 [04:50<03:24,  5.63it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  62%|██████▏   | 1852/3000 [04:51<02:51,  6.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  62%|██████▏   | 1854/3000 [04:51<03:23,  5.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  62%|██████▏   | 1856/3000 [04:52<03:35,  5.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  62%|██████▏   | 1857/3000 [04:52<03:47,  5.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  62%|██████▏   | 1859/3000 [04:52<03:07,  6.09it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  62%|██████▏   | 1863/3000 [04:52<02:23,  7.92it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  62%|██████▏   | 1866/3000 [04:53<03:48,  4.96it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  62%|██████▏   | 1868/3000 [04:54<03:03,  6.16it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  62%|██████▏   | 1870/3000 [04:54<03:17,  5.72it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  62%|██████▏   | 1873/3000 [04:54<03:02,  6.17it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  62%|██████▎   | 1875/3000 [04:55<02:20,  8.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  63%|██████▎   | 1877/3000 [04:55<02:54,  6.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  63%|██████▎   | 1878/3000 [04:56<04:12,  4.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  63%|██████▎   | 1880/3000 [04:56<03:46,  4.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  63%|██████▎   | 1882/3000 [04:56<03:28,  5.36it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  63%|██████▎   | 1885/3000 [04:57<02:54,  6.38it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  63%|██████▎   | 1886/3000 [04:57<03:26,  5.40it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  63%|██████▎   | 1889/3000 [04:57<03:03,  6.05it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  63%|██████▎   | 1891/3000 [04:57<02:26,  7.55it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  63%|██████▎   | 1893/3000 [04:59<05:22,  3.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  63%|██████▎   | 1895/3000 [04:59<04:13,  4.37it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  63%|██████▎   | 1896/3000 [04:59<04:20,  4.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  63%|██████▎   | 1899/3000 [05:00<03:16,  5.60it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  63%|██████▎   | 1900/3000 [05:00<03:15,  5.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  63%|██████▎   | 1904/3000 [05:00<03:17,  5.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  64%|██████▎   | 1908/3000 [05:01<01:54,  9.55it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  64%|██████▎   | 1910/3000 [05:01<02:51,  6.35it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  64%|██████▍   | 1913/3000 [05:02<02:34,  7.06it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  64%|██████▍   | 1914/3000 [05:02<03:11,  5.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  64%|██████▍   | 1916/3000 [05:03<04:17,  4.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  64%|██████▍   | 1918/3000 [05:03<03:26,  5.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  64%|██████▍   | 1920/3000 [05:03<02:58,  6.04it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  64%|██████▍   | 1923/3000 [05:04<02:27,  7.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  64%|██████▍   | 1924/3000 [05:04<03:20,  5.38it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  64%|██████▍   | 1925/3000 [05:04<03:52,  4.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  64%|██████▍   | 1928/3000 [05:05<03:10,  5.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  64%|██████▍   | 1930/3000 [05:05<02:46,  6.42it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  64%|██████▍   | 1931/3000 [05:05<02:41,  6.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  64%|██████▍   | 1934/3000 [05:06<02:47,  6.37it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  64%|██████▍   | 1935/3000 [05:06<02:55,  6.05it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  65%|██████▍   | 1938/3000 [05:07<03:32,  4.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  65%|██████▍   | 1940/3000 [05:07<02:46,  6.38it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  65%|██████▍   | 1942/3000 [05:07<02:13,  7.93it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  65%|██████▍   | 1947/3000 [05:08<02:19,  7.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  65%|██████▍   | 1949/3000 [05:08<03:00,  5.82it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  65%|██████▌   | 1951/3000 [05:08<02:24,  7.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  65%|██████▌   | 1953/3000 [05:09<03:17,  5.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  65%|██████▌   | 1956/3000 [05:10<03:42,  4.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  65%|██████▌   | 1959/3000 [05:10<03:36,  4.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  65%|██████▌   | 1962/3000 [05:11<02:20,  7.40it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  66%|██████▌   | 1967/3000 [05:11<01:59,  8.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  66%|██████▌   | 1969/3000 [05:12<02:51,  6.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  66%|██████▌   | 1971/3000 [05:12<03:30,  4.88it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  66%|██████▌   | 1974/3000 [05:13<02:48,  6.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  66%|██████▌   | 1977/3000 [05:13<02:34,  6.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  66%|██████▌   | 1980/3000 [05:13<02:05,  8.11it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  66%|██████▌   | 1982/3000 [05:14<02:03,  8.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  66%|██████▌   | 1983/3000 [05:14<02:34,  6.58it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  66%|██████▌   | 1985/3000 [05:14<02:59,  5.67it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  66%|██████▌   | 1986/3000 [05:15<03:48,  4.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  66%|██████▋   | 1990/3000 [05:16<03:16,  5.14it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  66%|██████▋   | 1992/3000 [05:16<02:50,  5.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  67%|██████▋   | 1996/3000 [05:16<02:12,  7.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  67%|██████▋   | 1999/3000 [05:17<02:37,  6.37it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  67%|██████▋   | 2002/3000 [05:18<03:23,  4.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  67%|██████▋   | 2005/3000 [05:18<02:42,  6.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  67%|██████▋   | 2008/3000 [05:19<02:27,  6.71it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  67%|██████▋   | 2011/3000 [05:19<02:24,  6.83it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  67%|██████▋   | 2013/3000 [05:19<01:51,  8.83it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  67%|██████▋   | 2015/3000 [05:20<03:16,  5.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  67%|██████▋   | 2016/3000 [05:20<03:30,  4.67it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  67%|██████▋   | 2017/3000 [05:21<04:54,  3.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  67%|██████▋   | 2019/3000 [05:21<04:08,  3.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  68%|██████▊   | 2025/3000 [05:22<02:15,  7.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  68%|██████▊   | 2029/3000 [05:22<01:44,  9.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  68%|██████▊   | 2031/3000 [05:22<02:14,  7.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  68%|██████▊   | 2033/3000 [05:23<03:15,  4.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  68%|██████▊   | 2037/3000 [05:23<02:14,  7.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  68%|██████▊   | 2039/3000 [05:24<02:22,  6.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  68%|██████▊   | 2041/3000 [05:24<02:19,  6.88it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  68%|██████▊   | 2042/3000 [05:24<02:49,  5.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  68%|██████▊   | 2043/3000 [05:25<02:55,  5.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  68%|██████▊   | 2044/3000 [05:25<03:02,  5.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  68%|██████▊   | 2046/3000 [05:25<03:20,  4.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  68%|██████▊   | 2049/3000 [05:26<02:33,  6.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  68%|██████▊   | 2050/3000 [05:26<02:48,  5.63it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metricsmetrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      " {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  68%|██████▊   | 2053/3000 [05:26<02:03,  7.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  68%|██████▊   | 2054/3000 [05:26<02:22,  6.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  68%|██████▊   | 2055/3000 [05:27<02:46,  5.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  69%|██████▊   | 2057/3000 [05:27<03:04,  5.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  69%|██████▊   | 2058/3000 [05:27<03:01,  5.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  69%|██████▊   | 2060/3000 [05:28<03:11,  4.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  69%|██████▉   | 2064/3000 [05:28<02:18,  6.78it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  69%|██████▉   | 2067/3000 [05:28<01:35,  9.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  69%|██████▉   | 2071/3000 [05:29<01:35,  9.77it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  69%|██████▉   | 2073/3000 [05:29<02:26,  6.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  69%|██████▉   | 2074/3000 [05:30<02:36,  5.93it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  69%|██████▉   | 2077/3000 [05:30<02:49,  5.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  69%|██████▉   | 2078/3000 [05:30<02:36,  5.88it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  69%|██████▉   | 2080/3000 [05:31<02:41,  5.71it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  69%|██████▉   | 2083/3000 [05:31<02:15,  6.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  70%|██████▉   | 2085/3000 [05:31<01:55,  7.92it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  70%|██████▉   | 2087/3000 [05:32<02:31,  6.05it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  70%|██████▉   | 2091/3000 [05:32<02:25,  6.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  70%|██████▉   | 2092/3000 [05:33<02:28,  6.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  70%|██████▉   | 2095/3000 [05:33<02:01,  7.46it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  70%|██████▉   | 2096/3000 [05:33<02:06,  7.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  70%|██████▉   | 2098/3000 [05:33<02:06,  7.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  70%|██████▉   | 2099/3000 [05:34<02:44,  5.46it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  70%|███████   | 2100/3000 [05:34<04:16,  3.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  70%|███████   | 2103/3000 [05:35<02:55,  5.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  70%|███████   | 2105/3000 [05:35<03:15,  4.57it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  70%|███████   | 2108/3000 [05:35<01:54,  7.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  70%|███████   | 2111/3000 [05:36<01:40,  8.87it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  70%|███████   | 2114/3000 [05:36<01:52,  7.89it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  71%|███████   | 2116/3000 [05:37<02:09,  6.80it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  71%|███████   | 2119/3000 [05:37<02:34,  5.72it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  71%|███████   | 2120/3000 [05:38<02:55,  5.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  71%|███████   | 2123/3000 [05:38<02:32,  5.74it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  71%|███████   | 2124/3000 [05:38<02:33,  5.71it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  71%|███████   | 2126/3000 [05:39<02:28,  5.88it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  71%|███████   | 2127/3000 [05:39<02:56,  4.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  71%|███████   | 2130/3000 [05:39<02:01,  7.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  71%|███████   | 2133/3000 [05:39<01:21, 10.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  71%|███████   | 2135/3000 [05:40<01:57,  7.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  71%|███████   | 2137/3000 [05:40<02:00,  7.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  71%|███████▏  | 2141/3000 [05:41<01:35,  8.96it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  71%|███████▏  | 2143/3000 [05:41<02:02,  6.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  72%|███████▏  | 2146/3000 [05:41<01:55,  7.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  72%|███████▏  | 2148/3000 [05:42<02:24,  5.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  72%|███████▏  | 2151/3000 [05:42<02:01,  6.96it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  72%|███████▏  | 2153/3000 [05:42<01:37,  8.66it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  72%|███████▏  | 2155/3000 [05:43<02:00,  7.04it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  72%|███████▏  | 2161/3000 [05:43<01:39,  8.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  72%|███████▏  | 2163/3000 [05:44<03:04,  4.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  72%|███████▏  | 2165/3000 [05:45<02:27,  5.67it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  72%|███████▏  | 2170/3000 [05:45<01:41,  8.16it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  72%|███████▏  | 2172/3000 [05:45<01:49,  7.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  73%|███████▎  | 2176/3000 [05:45<01:13, 11.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  73%|███████▎  | 2178/3000 [05:46<01:31,  8.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  73%|███████▎  | 2180/3000 [05:46<02:34,  5.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  73%|███████▎  | 2181/3000 [05:47<03:01,  4.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  73%|███████▎  | 2184/3000 [05:47<02:23,  5.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  73%|███████▎  | 2186/3000 [05:48<02:26,  5.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  73%|███████▎  | 2188/3000 [05:48<02:44,  4.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  73%|███████▎  | 2190/3000 [05:48<02:02,  6.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  73%|███████▎  | 2194/3000 [05:49<01:36,  8.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  73%|███████▎  | 2196/3000 [05:49<01:58,  6.78it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  73%|███████▎  | 2197/3000 [05:49<02:02,  6.55it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  73%|███████▎  | 2198/3000 [05:50<02:27,  5.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  73%|███████▎  | 2200/3000 [05:50<02:47,  4.77it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  73%|███████▎  | 2203/3000 [05:51<02:11,  6.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  74%|███████▎  | 2205/3000 [05:51<01:52,  7.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  74%|███████▎  | 2210/3000 [05:51<01:18, 10.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  74%|███████▎  | 2212/3000 [05:52<01:46,  7.37it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  74%|███████▍  | 2214/3000 [05:52<01:44,  7.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  74%|███████▍  | 2216/3000 [05:53<02:24,  5.42it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  74%|███████▍  | 2218/3000 [05:53<02:18,  5.63it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  74%|███████▍  | 2219/3000 [05:53<02:42,  4.80it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  74%|███████▍  | 2220/3000 [05:54<03:18,  3.93it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  74%|███████▍  | 2222/3000 [05:54<02:37,  4.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  74%|███████▍  | 2224/3000 [05:54<02:14,  5.77it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  74%|███████▍  | 2227/3000 [05:55<02:03,  6.28it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  74%|███████▍  | 2228/3000 [05:55<02:12,  5.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  74%|███████▍  | 2231/3000 [05:56<02:41,  4.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  74%|███████▍  | 2232/3000 [05:56<02:58,  4.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  74%|███████▍  | 2233/3000 [05:56<03:11,  4.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  75%|███████▍  | 2236/3000 [05:57<02:22,  5.37it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  75%|███████▍  | 2238/3000 [05:57<02:07,  5.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  75%|███████▍  | 2240/3000 [05:57<01:50,  6.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  75%|███████▍  | 2241/3000 [05:57<02:09,  5.86it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  75%|███████▍  | 2244/3000 [05:58<02:29,  5.06it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  75%|███████▍  | 2247/3000 [05:59<02:14,  5.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  75%|███████▍  | 2248/3000 [05:59<02:36,  4.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  75%|███████▍  | 2249/3000 [06:00<03:36,  3.47it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  75%|███████▌  | 2254/3000 [06:00<01:40,  7.42it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  75%|███████▌  | 2256/3000 [06:00<02:01,  6.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  75%|███████▌  | 2261/3000 [06:01<01:26,  8.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  75%|███████▌  | 2264/3000 [06:02<02:24,  5.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  76%|███████▌  | 2266/3000 [06:02<01:56,  6.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  76%|███████▌  | 2268/3000 [06:02<02:13,  5.47it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  76%|███████▌  | 2271/3000 [06:03<02:10,  5.57it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  76%|███████▌  | 2273/3000 [06:03<01:45,  6.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  76%|███████▌  | 2274/3000 [06:04<02:38,  4.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  76%|███████▌  | 2276/3000 [06:04<02:33,  4.72it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  76%|███████▌  | 2279/3000 [06:05<02:01,  5.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  76%|███████▌  | 2281/3000 [06:05<01:42,  7.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  76%|███████▌  | 2284/3000 [06:05<01:40,  7.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  76%|███████▌  | 2286/3000 [06:05<01:27,  8.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  76%|███████▌  | 2287/3000 [06:06<02:12,  5.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  76%|███████▋  | 2288/3000 [06:06<02:34,  4.61it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  76%|███████▋  | 2289/3000 [06:06<02:39,  4.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  76%|███████▋  | 2294/3000 [06:07<01:32,  7.61it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  77%|███████▋  | 2296/3000 [06:07<01:19,  8.86it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  77%|███████▋  | 2298/3000 [06:07<01:17,  9.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  77%|███████▋  | 2301/3000 [06:08<01:46,  6.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  77%|███████▋  | 2302/3000 [06:08<01:42,  6.84it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  77%|███████▋  | 2303/3000 [06:08<02:59,  3.88it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  77%|███████▋  | 2306/3000 [06:09<02:02,  5.67it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  77%|███████▋  | 2307/3000 [06:09<02:26,  4.74it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  77%|███████▋  | 2308/3000 [06:09<02:38,  4.37it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  77%|███████▋  | 2311/3000 [06:10<01:46,  6.46it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  77%|███████▋  | 2313/3000 [06:10<02:08,  5.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  77%|███████▋  | 2314/3000 [06:10<02:10,  5.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  77%|███████▋  | 2315/3000 [06:11<02:26,  4.66it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  77%|███████▋  | 2316/3000 [06:11<02:50,  4.02it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  77%|███████▋  | 2318/3000 [06:11<02:10,  5.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  77%|███████▋  | 2320/3000 [06:12<02:45,  4.11it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  77%|███████▋  | 2322/3000 [06:12<02:08,  5.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  78%|███████▊  | 2325/3000 [06:13<02:05,  5.37it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  78%|███████▊  | 2326/3000 [06:13<01:52,  5.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  78%|███████▊  | 2328/3000 [06:13<01:53,  5.92it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  78%|███████▊  | 2330/3000 [06:14<02:00,  5.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  78%|███████▊  | 2332/3000 [06:14<02:03,  5.42it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  78%|███████▊  | 2334/3000 [06:14<01:28,  7.52it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  78%|███████▊  | 2335/3000 [06:14<01:52,  5.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  78%|███████▊  | 2337/3000 [06:15<02:37,  4.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  78%|███████▊  | 2340/3000 [06:16<01:58,  5.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  78%|███████▊  | 2344/3000 [06:16<01:10,  9.37it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  78%|███████▊  | 2346/3000 [06:16<01:41,  6.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  78%|███████▊  | 2350/3000 [06:17<01:36,  6.72it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  78%|███████▊  | 2353/3000 [06:18<02:17,  4.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  78%|███████▊  | 2354/3000 [06:18<02:39,  4.05it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  79%|███████▊  | 2356/3000 [06:19<02:26,  4.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  79%|███████▊  | 2359/3000 [06:19<01:38,  6.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  79%|███████▊  | 2361/3000 [06:20<02:12,  4.82it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  79%|███████▊  | 2362/3000 [06:20<01:56,  5.49it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  79%|███████▉  | 2363/3000 [06:20<02:05,  5.09it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  79%|███████▉  | 2366/3000 [06:21<02:06,  5.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  79%|███████▉  | 2369/3000 [06:21<01:37,  6.47it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  79%|███████▉  | 2370/3000 [06:21<01:59,  5.28it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  79%|███████▉  | 2373/3000 [06:22<01:42,  6.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  79%|███████▉  | 2374/3000 [06:23<03:16,  3.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  79%|███████▉  | 2380/3000 [06:23<01:25,  7.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  79%|███████▉  | 2382/3000 [06:24<02:35,  3.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  79%|███████▉  | 2384/3000 [06:24<02:11,  4.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  80%|███████▉  | 2387/3000 [06:25<02:04,  4.92it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  80%|███████▉  | 2389/3000 [06:25<01:52,  5.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  80%|███████▉  | 2391/3000 [06:26<01:38,  6.17it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  80%|███████▉  | 2397/3000 [06:26<00:53, 11.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  80%|███████▉  | 2399/3000 [06:27<01:48,  5.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  80%|████████  | 2401/3000 [06:27<02:01,  4.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  80%|████████  | 2403/3000 [06:28<01:47,  5.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  80%|████████  | 2406/3000 [06:28<02:01,  4.89it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  80%|████████  | 2408/3000 [06:29<01:42,  5.78it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  80%|████████  | 2411/3000 [06:29<01:47,  5.46it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  80%|████████  | 2415/3000 [06:30<01:16,  7.67it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  81%|████████  | 2416/3000 [06:30<01:28,  6.57it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  81%|████████  | 2417/3000 [06:30<01:39,  5.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  81%|████████  | 2418/3000 [06:30<01:51,  5.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  81%|████████  | 2422/3000 [06:31<02:04,  4.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  81%|████████  | 2425/3000 [06:32<01:21,  7.05it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  81%|████████  | 2427/3000 [06:32<01:10,  8.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  81%|████████  | 2431/3000 [06:32<01:01,  9.27it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  81%|████████  | 2433/3000 [06:32<01:15,  7.55it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  81%|████████  | 2436/3000 [06:33<02:00,  4.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  81%|████████▏ | 2438/3000 [06:34<02:10,  4.32it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  81%|████████▏ | 2442/3000 [06:34<01:12,  7.74it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  82%|████████▏ | 2446/3000 [06:35<00:56,  9.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  82%|████████▏ | 2448/3000 [06:35<01:17,  7.09it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  82%|████████▏ | 2450/3000 [06:36<01:32,  5.93it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  82%|████████▏ | 2452/3000 [06:36<01:27,  6.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  82%|████████▏ | 2453/3000 [06:36<01:35,  5.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  82%|████████▏ | 2456/3000 [06:37<01:24,  6.47it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  82%|████████▏ | 2458/3000 [06:37<01:12,  7.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  82%|████████▏ | 2462/3000 [06:37<01:02,  8.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  82%|████████▏ | 2463/3000 [06:38<01:35,  5.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  82%|████████▏ | 2466/3000 [06:38<01:19,  6.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  82%|████████▏ | 2467/3000 [06:38<01:28,  6.04it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  82%|████████▏ | 2469/3000 [06:39<01:39,  5.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  82%|████████▏ | 2472/3000 [06:39<01:26,  6.09it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  82%|████████▎ | 2475/3000 [06:39<01:05,  8.05it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  83%|████████▎ | 2478/3000 [06:40<01:17,  6.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  83%|████████▎ | 2479/3000 [06:40<01:14,  7.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  83%|████████▎ | 2480/3000 [06:40<01:41,  5.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  83%|████████▎ | 2481/3000 [06:41<01:42,  5.04it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  83%|████████▎ | 2483/3000 [06:41<01:54,  4.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  83%|████████▎ | 2488/3000 [06:42<01:01,  8.32it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  83%|████████▎ | 2490/3000 [06:42<01:09,  7.35it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  83%|████████▎ | 2492/3000 [06:42<01:19,  6.37it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  83%|████████▎ | 2493/3000 [06:42<01:31,  5.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  83%|████████▎ | 2496/3000 [06:43<01:27,  5.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  83%|████████▎ | 2499/3000 [06:43<00:59,  8.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  83%|████████▎ | 2501/3000 [06:43<00:55,  9.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  83%|████████▎ | 2502/3000 [06:44<01:22,  6.02it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  83%|████████▎ | 2504/3000 [06:44<01:19,  6.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  84%|████████▎ | 2508/3000 [06:45<01:06,  7.35it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  84%|████████▎ | 2510/3000 [06:45<01:17,  6.36it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  84%|████████▍ | 2513/3000 [06:45<01:20,  6.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  84%|████████▍ | 2514/3000 [06:46<01:16,  6.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  84%|████████▍ | 2517/3000 [06:46<01:00,  7.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  84%|████████▍ | 2518/3000 [06:46<01:06,  7.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  84%|████████▍ | 2519/3000 [06:46<01:28,  5.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  84%|████████▍ | 2520/3000 [06:47<01:46,  4.49it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  84%|████████▍ | 2521/3000 [06:47<01:48,  4.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  84%|████████▍ | 2524/3000 [06:48<01:51,  4.27it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  84%|████████▍ | 2528/3000 [06:48<01:05,  7.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  84%|████████▍ | 2532/3000 [06:49<00:50,  9.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  84%|████████▍ | 2534/3000 [06:49<01:03,  7.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  84%|████████▍ | 2535/3000 [06:49<01:30,  5.14it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  85%|████████▍ | 2536/3000 [06:50<01:31,  5.05it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  85%|████████▍ | 2539/3000 [06:50<01:24,  5.47it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  85%|████████▍ | 2541/3000 [06:51<01:24,  5.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  85%|████████▍ | 2544/3000 [06:51<00:59,  7.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  85%|████████▍ | 2547/3000 [06:51<01:01,  7.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  85%|████████▌ | 2550/3000 [06:52<00:53,  8.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  85%|████████▌ | 2552/3000 [06:52<00:46,  9.57it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  85%|████████▌ | 2557/3000 [06:53<00:56,  7.84it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  85%|████████▌ | 2561/3000 [06:53<00:58,  7.57it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  85%|████████▌ | 2563/3000 [06:53<00:47,  9.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  86%|████████▌ | 2565/3000 [06:54<01:17,  5.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  86%|████████▌ | 2567/3000 [06:54<01:20,  5.40it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  86%|████████▌ | 2569/3000 [06:55<01:16,  5.63it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  86%|████████▌ | 2570/3000 [06:55<01:11,  6.02it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  86%|████████▌ | 2573/3000 [06:55<00:58,  7.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  86%|████████▌ | 2577/3000 [06:56<00:49,  8.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  86%|████████▌ | 2579/3000 [06:56<00:50,  8.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  86%|████████▌ | 2581/3000 [06:56<00:46,  9.04it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  86%|████████▌ | 2583/3000 [06:57<01:25,  4.86it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  86%|████████▌ | 2586/3000 [06:57<01:12,  5.67it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  86%|████████▌ | 2587/3000 [06:57<01:07,  6.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  86%|████████▋ | 2589/3000 [06:58<01:00,  6.80it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  86%|████████▋ | 2591/3000 [06:58<01:05,  6.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  86%|████████▋ | 2595/3000 [06:59<00:51,  7.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  87%|████████▋ | 2597/3000 [06:59<01:01,  6.55it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  87%|████████▋ | 2598/3000 [06:59<01:11,  5.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  87%|████████▋ | 2601/3000 [07:00<01:08,  5.82it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  87%|████████▋ | 2603/3000 [07:00<00:51,  7.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  87%|████████▋ | 2607/3000 [07:01<01:22,  4.78it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  87%|████████▋ | 2608/3000 [07:01<01:13,  5.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  87%|████████▋ | 2610/3000 [07:02<01:12,  5.38it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  87%|████████▋ | 2613/3000 [07:02<00:59,  6.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  87%|████████▋ | 2615/3000 [07:02<00:50,  7.67it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  87%|████████▋ | 2621/3000 [07:03<00:28, 13.11it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  87%|████████▋ | 2623/3000 [07:03<01:04,  5.86it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  88%|████████▊ | 2625/3000 [07:04<01:01,  6.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  88%|████████▊ | 2627/3000 [07:04<01:02,  5.93it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  88%|████████▊ | 2628/3000 [07:05<01:20,  4.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  88%|████████▊ | 2629/3000 [07:05<01:23,  4.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  88%|████████▊ | 2633/3000 [07:05<00:55,  6.67it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  88%|████████▊ | 2636/3000 [07:05<00:48,  7.55it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  88%|████████▊ | 2637/3000 [07:06<00:49,  7.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  88%|████████▊ | 2641/3000 [07:06<00:50,  7.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  88%|████████▊ | 2642/3000 [07:06<00:54,  6.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  88%|████████▊ | 2645/3000 [07:07<00:50,  7.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  88%|████████▊ | 2648/3000 [07:07<00:42,  8.37it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  88%|████████▊ | 2651/3000 [07:08<00:44,  7.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  88%|████████▊ | 2653/3000 [07:08<00:48,  7.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  89%|████████▊ | 2656/3000 [07:08<00:38,  8.82it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  89%|████████▊ | 2657/3000 [07:08<00:49,  6.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  89%|████████▊ | 2658/3000 [07:09<01:03,  5.37it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  89%|████████▊ | 2660/3000 [07:09<01:06,  5.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  89%|████████▊ | 2661/3000 [07:10<01:14,  4.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  89%|████████▊ | 2662/3000 [07:10<01:15,  4.46it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  89%|████████▉ | 2664/3000 [07:10<01:17,  4.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  89%|████████▉ | 2667/3000 [07:11<00:56,  5.88it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  89%|████████▉ | 2668/3000 [07:11<00:53,  6.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  89%|████████▉ | 2669/3000 [07:11<01:20,  4.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  89%|████████▉ | 2671/3000 [07:12<01:11,  4.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  89%|████████▉ | 2674/3000 [07:12<00:55,  5.92it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  89%|████████▉ | 2676/3000 [07:12<00:48,  6.67it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  89%|████████▉ | 2681/3000 [07:13<00:30, 10.36it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  89%|████████▉ | 2683/3000 [07:13<00:33,  9.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  90%|████████▉ | 2685/3000 [07:13<00:35,  8.84it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  90%|████████▉ | 2686/3000 [07:14<00:45,  6.92it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  90%|████████▉ | 2688/3000 [07:14<00:43,  7.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  90%|████████▉ | 2691/3000 [07:15<00:53,  5.82it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  90%|████████▉ | 2693/3000 [07:15<00:58,  5.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  90%|████████▉ | 2695/3000 [07:15<00:52,  5.82it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  90%|████████▉ | 2697/3000 [07:16<00:48,  6.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  90%|████████▉ | 2699/3000 [07:16<00:42,  7.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  90%|█████████ | 2702/3000 [07:16<00:37,  7.84it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  90%|█████████ | 2703/3000 [07:17<01:18,  3.77it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  90%|█████████ | 2704/3000 [07:17<01:15,  3.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  90%|█████████ | 2706/3000 [07:17<00:58,  5.04it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  90%|█████████ | 2709/3000 [07:18<00:41,  6.96it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  90%|█████████ | 2711/3000 [07:18<00:40,  7.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  91%|█████████ | 2716/3000 [07:18<00:24, 11.40it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  91%|█████████ | 2718/3000 [07:19<00:46,  6.04it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  91%|█████████ | 2719/3000 [07:20<01:08,  4.09it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  91%|█████████ | 2723/3000 [07:20<00:43,  6.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  91%|█████████ | 2726/3000 [07:20<00:30,  8.96it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  91%|█████████ | 2728/3000 [07:21<00:33,  8.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  91%|█████████ | 2730/3000 [07:21<00:32,  8.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  91%|█████████ | 2732/3000 [07:21<00:36,  7.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  91%|█████████ | 2734/3000 [07:22<00:42,  6.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  91%|█████████ | 2735/3000 [07:22<00:45,  5.83it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  91%|█████████ | 2736/3000 [07:22<00:47,  5.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  91%|█████████▏| 2739/3000 [07:23<00:45,  5.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  91%|█████████▏| 2743/3000 [07:23<00:31,  8.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  92%|█████████▏| 2745/3000 [07:23<00:29,  8.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  92%|█████████▏| 2747/3000 [07:24<00:34,  7.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  92%|█████████▏| 2750/3000 [07:24<00:30,  8.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  92%|█████████▏| 2753/3000 [07:25<00:42,  5.83it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  92%|█████████▏| 2754/3000 [07:25<00:40,  6.14it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  92%|█████████▏| 2755/3000 [07:25<00:44,  5.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  92%|█████████▏| 2758/3000 [07:25<00:33,  7.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  92%|█████████▏| 2759/3000 [07:25<00:32,  7.49it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  92%|█████████▏| 2761/3000 [07:26<00:40,  5.92it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  92%|█████████▏| 2764/3000 [07:26<00:29,  8.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  92%|█████████▏| 2768/3000 [07:27<00:40,  5.71it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  92%|█████████▏| 2771/3000 [07:27<00:26,  8.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  92%|█████████▏| 2773/3000 [07:28<00:42,  5.36it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  93%|█████████▎| 2777/3000 [07:28<00:31,  7.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  93%|█████████▎| 2779/3000 [07:29<00:29,  7.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  93%|█████████▎| 2781/3000 [07:29<00:29,  7.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  93%|█████████▎| 2783/3000 [07:29<00:31,  6.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  93%|█████████▎| 2785/3000 [07:30<00:33,  6.38it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  93%|█████████▎| 2787/3000 [07:30<00:31,  6.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  93%|█████████▎| 2788/3000 [07:30<00:37,  5.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  93%|█████████▎| 2789/3000 [07:30<00:40,  5.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  93%|█████████▎| 2791/3000 [07:31<00:43,  4.77it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  93%|█████████▎| 2795/3000 [07:31<00:30,  6.77it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  93%|█████████▎| 2797/3000 [07:32<00:26,  7.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  93%|█████████▎| 2800/3000 [07:32<00:24,  8.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  93%|█████████▎| 2801/3000 [07:32<00:29,  6.82it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  93%|█████████▎| 2802/3000 [07:32<00:31,  6.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  93%|█████████▎| 2804/3000 [07:33<00:32,  5.96it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  94%|█████████▎| 2805/3000 [07:33<00:58,  3.36it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  94%|█████████▎| 2811/3000 [07:34<00:33,  5.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  94%|█████████▍| 2815/3000 [07:35<00:25,  7.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  94%|█████████▍| 2818/3000 [07:35<00:18,  9.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  94%|█████████▍| 2821/3000 [07:35<00:15, 11.49it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  94%|█████████▍| 2823/3000 [07:36<00:37,  4.66it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  94%|█████████▍| 2826/3000 [07:37<00:39,  4.36it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  94%|█████████▍| 2830/3000 [07:37<00:25,  6.67it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  94%|█████████▍| 2834/3000 [07:38<00:22,  7.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  95%|█████████▍| 2836/3000 [07:38<00:24,  6.74it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  95%|█████████▍| 2839/3000 [07:38<00:18,  8.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  95%|█████████▍| 2842/3000 [07:40<00:36,  4.36it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  95%|█████████▍| 2844/3000 [07:40<00:27,  5.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  95%|█████████▍| 2847/3000 [07:40<00:26,  5.88it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  95%|█████████▌| 2852/3000 [07:41<00:15,  9.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  95%|█████████▌| 2854/3000 [07:41<00:20,  7.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  95%|█████████▌| 2856/3000 [07:42<00:25,  5.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  95%|█████████▌| 2857/3000 [07:42<00:26,  5.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  95%|█████████▌| 2859/3000 [07:42<00:25,  5.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  95%|█████████▌| 2861/3000 [07:43<00:24,  5.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  96%|█████████▌| 2865/3000 [07:43<00:19,  6.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  96%|█████████▌| 2867/3000 [07:44<00:19,  6.71it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  96%|█████████▌| 2868/3000 [07:44<00:18,  7.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  96%|█████████▌| 2870/3000 [07:44<00:18,  7.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  96%|█████████▌| 2872/3000 [07:44<00:18,  6.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  96%|█████████▌| 2873/3000 [07:45<00:23,  5.36it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  96%|█████████▌| 2874/3000 [07:45<00:28,  4.42it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  96%|█████████▌| 2877/3000 [07:45<00:19,  6.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  96%|█████████▌| 2881/3000 [07:46<00:12,  9.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  96%|█████████▌| 2883/3000 [07:46<00:13,  8.47it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  96%|█████████▌| 2886/3000 [07:46<00:12,  9.47it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  96%|█████████▋| 2888/3000 [07:46<00:11,  9.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  96%|█████████▋| 2890/3000 [07:47<00:20,  5.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  96%|█████████▋| 2893/3000 [07:48<00:16,  6.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  97%|█████████▋| 2896/3000 [07:49<00:25,  4.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  97%|█████████▋| 2899/3000 [07:49<00:15,  6.37it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metricsmetrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      " {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  97%|█████████▋| 2903/3000 [07:49<00:12,  7.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  97%|█████████▋| 2905/3000 [07:50<00:15,  6.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  97%|█████████▋| 2907/3000 [07:50<00:14,  6.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  97%|█████████▋| 2909/3000 [07:51<00:18,  4.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  97%|█████████▋| 2912/3000 [07:51<00:13,  6.36it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  97%|█████████▋| 2914/3000 [07:51<00:13,  6.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  97%|█████████▋| 2919/3000 [07:52<00:06, 12.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  97%|█████████▋| 2921/3000 [07:52<00:09,  8.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  97%|█████████▋| 2923/3000 [07:52<00:10,  7.36it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  97%|█████████▋| 2924/3000 [07:53<00:11,  6.66it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  98%|█████████▊| 2927/3000 [07:53<00:14,  5.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  98%|█████████▊| 2931/3000 [07:54<00:09,  6.93it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  98%|█████████▊| 2934/3000 [07:54<00:09,  7.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  98%|█████████▊| 2939/3000 [07:54<00:05, 12.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  98%|█████████▊| 2941/3000 [07:55<00:06,  9.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  98%|█████████▊| 2943/3000 [07:55<00:07,  7.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  98%|█████████▊| 2945/3000 [07:56<00:08,  6.14it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  98%|█████████▊| 2948/3000 [07:56<00:08,  5.88it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  98%|█████████▊| 2951/3000 [07:56<00:06,  8.16it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  98%|█████████▊| 2953/3000 [07:57<00:06,  7.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  99%|█████████▊| 2957/3000 [07:57<00:05,  7.61it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  99%|█████████▊| 2959/3000 [07:57<00:04,  9.11it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  99%|█████████▊| 2961/3000 [07:58<00:04,  8.60it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  99%|█████████▉| 2963/3000 [07:58<00:06,  5.36it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  99%|█████████▉| 2964/3000 [07:59<00:07,  5.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  99%|█████████▉| 2965/3000 [07:59<00:07,  4.96it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  99%|█████████▉| 2966/3000 [07:59<00:07,  4.74it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  99%|█████████▉| 2968/3000 [07:59<00:06,  5.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  99%|█████████▉| 2970/3000 [08:00<00:04,  7.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  99%|█████████▉| 2971/3000 [08:00<00:05,  5.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  99%|█████████▉| 2973/3000 [08:00<00:04,  5.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  99%|█████████▉| 2976/3000 [08:01<00:03,  6.92it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  99%|█████████▉| 2979/3000 [08:01<00:02, 10.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  99%|█████████▉| 2982/3000 [08:01<00:02,  8.72it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  99%|█████████▉| 2984/3000 [08:02<00:02,  5.49it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow: 100%|█████████▉| 2985/3000 [08:02<00:02,  5.37it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow: 100%|█████████▉| 2989/3000 [08:02<00:01,  6.71it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow: 100%|█████████▉| 2991/3000 [08:03<00:01,  7.07it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow: 100%|█████████▉| 2994/3000 [08:03<00:00,  8.04it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow: 100%|█████████▉| 2996/3000 [08:03<00:00,  6.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow: 100%|█████████▉| 2998/3000 [08:04<00:00,  4.92it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow: 100%|█████████▉| 2999/3000 [08:05<00:00,  4.05it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow: 100%|██████████| 3000/3000 [08:06<00:00,  6.17it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "\u001b[32m2026-01-03 19:26:42.546\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36m__main__\u001b[0m:\u001b[36m<module>\u001b[0m:\u001b[36m3\u001b[0m - \u001b[1mEvaluation metrics (after optimization): {'f1': 0.087, 'em': 0.087, 'acc': 0.726}\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "with suppress_logger_info():\n",
    "    results = textgrad_optimizer.evaluate(dataset=benchmark, eval_mode=\"test\")\n",
    "logger.info(f\"Evaluation metrics (after optimization): {results}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "2595c93b",
   "metadata": {},
   "outputs": [],
   "source": [
    "# metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a4af1fcb",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "b463a4a5",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "\n",
    "from dotenv import load_dotenv\n",
    "\n",
    "from evoagentx.agents.agent_manager import AgentManager\n",
    "from evoagentx.benchmark import HotPotQA\n",
    "from evoagentx.core.callbacks import suppress_logger_info\n",
    "from evoagentx.core.logging import logger\n",
    "from evoagentx.evaluators import Evaluator\n",
    "from evoagentx.models import OpenAILLM, OpenAILLMConfig\n",
    "from evoagentx.optimizers import TextGradOptimizer\n",
    "from evoagentx.prompts import StringTemplate\n",
    "from evoagentx.workflow import SequentialWorkFlowGraph\n",
    "from dotenv import load_dotenv\n",
    "\n",
    "from evoagentx.agents.agent_manager import AgentManager\n",
    "from evoagentx.benchmark import MBPP\n",
    "from evoagentx.core.callbacks import suppress_logger_info\n",
    "from evoagentx.core.logging import logger\n",
    "from evoagentx.evaluators import Evaluator\n",
    "from evoagentx.models import OpenAILLM, OpenAILLMConfig\n",
    "from evoagentx.optimizers import TextGradOptimizer\n",
    "from evoagentx.prompts import StringTemplate\n",
    "from evoagentx.workflow import SequentialWorkFlowGraph\n",
    "\n",
    "from evoagentx.models import OpenAILLMConfig, OpenAILLM\n",
    "from evoagentx.workflow import SEWWorkFlowGraph, STRUCTUREWorkFlowGraph\n",
    "from evoagentx.agents import AgentManager\n",
    "from evoagentx.benchmark import HumanEval,AFlowMBPP\n",
    "from evoagentx.evaluators import Evaluator \n",
    "from evoagentx.optimizers import SEWOptimizer, STRUCTUREOptimizer\n",
    "from evoagentx.optimizers.structure_optimizer import STRUCTUREWorkFlowScheme\n",
    "from evoagentx.core.callbacks import suppress_logger_info\n",
    "\n",
    "from evoagentx.models import OpenAILLMConfig, OpenAILLM,AzureOpenAIConfig,LiteLLMConfig,LiteLLM\n",
    "from evoagentx.workflow import SEWWorkFlowGraph \n",
    "from evoagentx.agents import AgentManager\n",
    "from evoagentx.benchmark import MBPPPLUS, AFlowMBPPPLUS\n",
    "from evoagentx.evaluators import Evaluator \n",
    "from evoagentx.optimizers import SEWOptimizer \n",
    "from evoagentx.core.callbacks import suppress_logger_info\n",
    "from evoagentx.benchmark import HumanEvalPLUS\n",
    "from evoagentx.benchmark import SciCode\n",
    "from copy import deepcopy\n",
    "\n",
    "import nest_asyncio\n",
    "nest_asyncio.apply()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "480de58c",
   "metadata": {},
   "outputs": [],
   "source": [
    "from evoagentx.benchmark import PertQA\n",
    "from copy import deepcopy\n",
    "\n",
    "import nest_asyncio\n",
    "nest_asyncio.apply()\n",
    "\n",
    "def collate_func(example: dict) -> dict:\n",
    "    problem = \"Question: {}\\n\\nAnswer:\".format(example[\"question\"])\n",
    "    return {\"problem\": problem}\n",
    "\n",
    "\n",
    "hotpotqa_graph_data = {\n",
    "    \"goal\": \"Answer the question based on the context. The answer should be a direct response to the question, without including explanations or reasoning.\",\n",
    "    \"tasks\": [\n",
    "        {\n",
    "            \"name\": \"answer_generate\",\n",
    "            \"description\": \"Answer the question based on the context.\",\n",
    "            \"inputs\": [\n",
    "                {\"name\": \"problem\", \"type\": \"str\", \"required\": True, \"description\": \"The problem to solve.\"}\n",
    "            ],\n",
    "            \"outputs\": [\n",
    "                {\"name\": \"answer\", \"type\": \"str\", \"required\": True, \"description\": \"The answer to the problem.\"}\n",
    "            ],\n",
    "            \"prompt_template\": StringTemplate(instruction=\"Think step by step to answer the question. You should explain your thinking process in the 'thought' field, and provide the final answer in the 'answer' field.\\nFormat your output in xml format, such as <thought>xxx</thought> and <answer>xxx</answer>.\"),\n",
    "            \"parse_mode\": \"xml\"\n",
    "        }\n",
    "    ] \n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "e9818857",
   "metadata": {},
   "outputs": [],
   "source": [
    "# os.environ[\"AZURE_OPENAI_DEPLOYMENT_NAME\"] = \"gpt-4o-mini\"\n",
    "# os.environ[\"AZURE_OPENAI_ENDPOINT\"] = \"https://optimizehumaneval.cognitiveservices.azure.com/\"\n",
    "# os.environ[\"AZURE_OPENAI_KEY\"] = \"2b7h6anDXRsl5XHDUAGKHpjh3DLv9kLjcjGXN6PvsEmLVf1i3imMJQQJ99BKACYeBjFXJ3w3AAABACOGATqP\"\n",
    "# os.environ[\"AZURE_OPENAI_API_VERSION\"] = \"2025-01-01-preview\"\n",
    "# llm_config = LiteLLMConfig(model=\"azure/\" + os.getenv(\"AZURE_OPENAI_DEPLOYMENT_NAME\"),  # Azure model format\n",
    "#     azure_endpoint=os.getenv(\"AZURE_OPENAI_ENDPOINT\"),\n",
    "#     azure_key=os.getenv(\"AZURE_OPENAI_KEY\"),\n",
    "#     api_version=os.getenv(\"AZURE_OPENAI_API_VERSION\", \"2024-12-01-preview\"), top_p=0.85, temperature=0.2, frequency_penalty=0.0, presence_penalty=0.0)\n",
    "\n",
    "# executor_llm = LiteLLM(config=llm_config)\n",
    "# optimizer_llm = LiteLLM(config=llm_config)\n",
    "\n",
    "os.environ[\"AZURE_OPENAI_DEPLOYMENT_NAME\"] = \"gpt-4o-mini\"\n",
    "os.environ[\"AZURE_OPENAI_ENDPOINT\"] = \"https://tianyuliu-hua-raredisea-resource.cognitiveservices.azure.com/\"\n",
    "os.environ[\"AZURE_OPENAI_KEY\"] = \"2pa9h2ZIN1lQepFWwYADlXIKIansa9KPhxMoumeGbRQ08f2uDTXiJQQJ99BKACHYHv6XJ3w3AAAAACOGsQIt\"\n",
    "os.environ[\"AZURE_OPENAI_API_VERSION\"] = \"2025-01-01-preview\"\n",
    "llm_config = LiteLLMConfig(model=\"azure/\" + os.getenv(\"AZURE_OPENAI_DEPLOYMENT_NAME\"),  # Azure model format\n",
    "    azure_endpoint=os.getenv(\"AZURE_OPENAI_ENDPOINT\"),\n",
    "    azure_key=os.getenv(\"AZURE_OPENAI_KEY\"),\n",
    "    api_version=os.getenv(\"AZURE_OPENAI_API_VERSION\", \"2024-12-01-preview\"), top_p=0.85, temperature=0.2, frequency_penalty=0.0, presence_penalty=0.0)\n",
    "\n",
    "executor_llm = LiteLLM(config=llm_config)\n",
    "optimizer_llm = LiteLLM(config=llm_config)\n",
    "llm = executor_llm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "796739dc",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2026-01-04 10:27:46.950\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.benchmark.pertqa\u001b[0m:\u001b[36m_load_data_from_file\u001b[0m:\u001b[36m52\u001b[0m - \u001b[1mloading HotPotQA data from /home/tl688/pitl688/selfevolve/EvoAgentX/examples/pertqa/reploge_train.json ...\u001b[0m\n",
      "\u001b[32m2026-01-04 10:27:47.016\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.benchmark.pertqa\u001b[0m:\u001b[36m_load_data_from_file\u001b[0m:\u001b[36m52\u001b[0m - \u001b[1mloading HotPotQA data from /home/tl688/pitl688/selfevolve/EvoAgentX/examples/pertqa/reploge_train.json ...\u001b[0m\n",
      "\u001b[32m2026-01-04 10:27:47.081\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.benchmark.pertqa\u001b[0m:\u001b[36m_load_data_from_file\u001b[0m:\u001b[36m52\u001b[0m - \u001b[1mloading HotPotQA data from /home/tl688/pitl688/selfevolve/EvoAgentX/examples/pertqa/reploge_test.json ...\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "benchmark = PertQA(pertdata='reploge')\n",
    "workflow_graph = SequentialWorkFlowGraph.from_dict(hotpotqa_graph_data)\n",
    "agent_manager = AgentManager()\n",
    "agent_manager.add_agents_from_workflow(workflow_graph, executor_llm.config)\n",
    "\n",
    "evaluator = Evaluator(\n",
    "    llm=executor_llm, \n",
    "    agent_manager=agent_manager, \n",
    "    collate_func=collate_func, \n",
    "    num_workers=20, \n",
    "    verbose=True\n",
    ")\n",
    "\n",
    "textgrad_optimizer = TextGradOptimizer(\n",
    "    graph=workflow_graph, \n",
    "    optimize_mode=\"all\",\n",
    "    executor_llm=executor_llm, \n",
    "    optimizer_llm=optimizer_llm,\n",
    "    batch_size=3,\n",
    "    max_steps=20,\n",
    "    evaluator=evaluator,\n",
    "    eval_every_n_steps=1,\n",
    "    eval_rounds=1,\n",
    "    save_interval=None,\n",
    "    save_path=\"./\",\n",
    "    rollback=True,\n",
    "    constraints=[]\n",
    ")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "baa44bb7",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "ename": "AttributeError",
     "evalue": "'PertQA' object has no attribute '_fulldata'",
     "output_type": "error",
     "traceback": [
      "\u001b[31m---------------------------------------------------------------------------\u001b[39m",
      "\u001b[31mAttributeError\u001b[39m                            Traceback (most recent call last)",
      "\u001b[36mCell\u001b[39m\u001b[36m \u001b[39m\u001b[32mIn[23]\u001b[39m\u001b[32m, line 1\u001b[39m\n\u001b[32m----> \u001b[39m\u001b[32m1\u001b[39m \u001b[38;5;28mlen\u001b[39m(\u001b[43mbenchmark\u001b[49m\u001b[43m.\u001b[49m\u001b[43m_fulldata\u001b[49m)\n",
      "\u001b[31mAttributeError\u001b[39m: 'PertQA' object has no attribute '_fulldata'"
     ]
    }
   ],
   "source": [
    "len(benchmark._fulldata)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "3ed1f571",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2026-01-04 10:27:48.819\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36m__main__\u001b[0m:\u001b[36m<module>\u001b[0m:\u001b[36m1\u001b[0m - \u001b[1mEvaluating workflow on test set...\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   0%|          | 1/3000 [00:01<1:21:35,  1.63s/it]Task exception was never retrieved\n",
      "future: <Task finished name='Task-21119' coro=<_client_async_logging_helper() done, defined at /gpfs/radev/home/tl688/.conda/envs/evoagentx/lib/python3.11/site-packages/litellm/utils.py:855> exception=RuntimeError('Event loop is closed')>\n",
      "Traceback (most recent call last):\n",
      "  File \"/gpfs/radev/home/tl688/.conda/envs/evoagentx/lib/python3.11/asyncio/tasks.py\", line 277, in __step\n",
      "    result = coro.send(None)\n",
      "             ^^^^^^^^^^^^^^^\n",
      "  File \"/gpfs/radev/home/tl688/.conda/envs/evoagentx/lib/python3.11/site-packages/litellm/utils.py\", line 873, in _client_async_logging_helper\n",
      "    GLOBAL_LOGGING_WORKER.ensure_initialized_and_enqueue(\n",
      "  File \"/gpfs/radev/home/tl688/.conda/envs/evoagentx/lib/python3.11/site-packages/litellm/litellm_core_utils/logging_worker.py\", line 322, in ensure_initialized_and_enqueue\n",
      "    self.enqueue(async_coroutine)\n",
      "  File \"/gpfs/radev/home/tl688/.conda/envs/evoagentx/lib/python3.11/site-packages/litellm/litellm_core_utils/logging_worker.py\", line 131, in enqueue\n",
      "    self._queue.put_nowait(task)\n",
      "  File \"/gpfs/radev/home/tl688/.conda/envs/evoagentx/lib/python3.11/asyncio/queues.py\", line 147, in put_nowait\n",
      "    self._wakeup_next(self._getters)\n",
      "  File \"/gpfs/radev/home/tl688/.conda/envs/evoagentx/lib/python3.11/asyncio/queues.py\", line 63, in _wakeup_next\n",
      "    waiter.set_result(None)\n",
      "  File \"/gpfs/radev/home/tl688/.conda/envs/evoagentx/lib/python3.11/asyncio/futures.py\", line 263, in set_result\n",
      "    self.__schedule_callbacks()\n",
      "  File \"/gpfs/radev/home/tl688/.conda/envs/evoagentx/lib/python3.11/asyncio/futures.py\", line 173, in __schedule_callbacks\n",
      "    self._loop.call_soon(callback, self, context=ctx)\n",
      "  File \"/gpfs/radev/home/tl688/.conda/envs/evoagentx/lib/python3.11/asyncio/base_events.py\", line 762, in call_soon\n",
      "    self._check_closed()\n",
      "  File \"/gpfs/radev/home/tl688/.conda/envs/evoagentx/lib/python3.11/asyncio/base_events.py\", line 520, in _check_closed\n",
      "    raise RuntimeError('Event loop is closed')\n",
      "RuntimeError: Event loop is closed\n",
      "Evaluating workflow:   0%|          | 2/3000 [00:01<37:28,  1.33it/s]  "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   0%|          | 8/3000 [00:02<06:59,  7.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   0%|          | 15/3000 [00:02<03:23, 14.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   1%|          | 19/3000 [00:02<02:57, 16.78it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   1%|          | 22/3000 [00:03<06:46,  7.32it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   1%|          | 24/3000 [00:03<06:24,  7.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   1%|          | 26/3000 [00:03<06:14,  7.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   1%|          | 28/3000 [00:04<05:47,  8.55it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   1%|▏         | 38/3000 [00:04<02:42, 18.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   1%|▏         | 41/3000 [00:04<04:51, 10.14it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   1%|▏         | 43/3000 [00:05<05:28,  9.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   2%|▏         | 45/3000 [00:05<06:22,  7.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   2%|▏         | 47/3000 [00:05<06:10,  7.96it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   2%|▏         | 54/3000 [00:06<04:08, 11.88it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   2%|▏         | 57/3000 [00:06<03:28, 14.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   2%|▏         | 59/3000 [00:06<03:44, 13.07it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   2%|▏         | 62/3000 [00:07<05:25,  9.02it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   2%|▏         | 68/3000 [00:07<04:20, 11.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   2%|▏         | 70/3000 [00:07<04:41, 10.42it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   3%|▎         | 76/3000 [00:08<03:43, 13.09it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   3%|▎         | 80/3000 [00:08<06:15,  7.77it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   3%|▎         | 84/3000 [00:09<05:18,  9.16it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   3%|▎         | 86/3000 [00:09<04:32, 10.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   3%|▎         | 90/3000 [00:09<03:41, 13.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   3%|▎         | 94/3000 [00:09<03:50, 12.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   3%|▎         | 96/3000 [00:10<03:52, 12.48it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   3%|▎         | 98/3000 [00:10<06:56,  6.97it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   3%|▎         | 102/3000 [00:11<06:02,  8.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   4%|▎         | 107/3000 [00:11<04:00, 12.04it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   4%|▎         | 109/3000 [00:11<04:02, 11.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   4%|▍         | 115/3000 [00:12<03:50, 12.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   4%|▍         | 117/3000 [00:12<03:38, 13.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   4%|▍         | 119/3000 [00:12<06:59,  6.86it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   4%|▍         | 122/3000 [00:13<05:32,  8.67it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   4%|▍         | 124/3000 [00:13<05:40,  8.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   4%|▍         | 131/3000 [00:13<03:25, 13.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   4%|▍         | 133/3000 [00:13<04:17, 11.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   4%|▍         | 135/3000 [00:14<05:26,  8.78it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   5%|▍         | 141/3000 [00:15<05:01,  9.48it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   5%|▍         | 143/3000 [00:15<04:50,  9.82it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   5%|▍         | 145/3000 [00:15<04:24, 10.77it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   5%|▍         | 149/3000 [00:15<04:43, 10.04it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   5%|▌         | 151/3000 [00:15<04:13, 11.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   5%|▌         | 153/3000 [00:16<05:30,  8.61it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   5%|▌         | 155/3000 [00:16<04:58,  9.52it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   5%|▌         | 157/3000 [00:16<05:37,  8.42it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   5%|▌         | 159/3000 [00:17<06:05,  7.77it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   5%|▌         | 163/3000 [00:17<05:34,  8.49it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   6%|▌         | 166/3000 [00:17<05:02,  9.37it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   6%|▌         | 170/3000 [00:18<04:08, 11.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   6%|▌         | 172/3000 [00:18<04:37, 10.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   6%|▌         | 175/3000 [00:18<05:19,  8.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   6%|▌         | 178/3000 [00:19<05:08,  9.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   6%|▌         | 182/3000 [00:19<04:16, 11.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   6%|▌         | 185/3000 [00:19<05:44,  8.17it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   6%|▌         | 187/3000 [00:20<05:37,  8.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   6%|▋         | 192/3000 [00:20<04:16, 10.96it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   7%|▋         | 196/3000 [00:20<04:19, 10.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   7%|▋         | 198/3000 [00:20<03:51, 12.09it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   7%|▋         | 200/3000 [00:21<03:50, 12.16it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   7%|▋         | 202/3000 [00:21<07:44,  6.02it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   7%|▋         | 209/3000 [00:22<03:59, 11.67it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   7%|▋         | 211/3000 [00:22<04:40,  9.93it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   7%|▋         | 213/3000 [00:22<04:42,  9.88it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   7%|▋         | 217/3000 [00:22<04:21, 10.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   7%|▋         | 221/3000 [00:23<04:41,  9.88it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   7%|▋         | 223/3000 [00:23<04:41,  9.86it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   8%|▊         | 225/3000 [00:23<04:28, 10.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   8%|▊         | 229/3000 [00:23<03:42, 12.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   8%|▊         | 234/3000 [00:24<04:07, 11.17it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   8%|▊         | 238/3000 [00:24<03:24, 13.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   8%|▊         | 242/3000 [00:25<03:51, 11.93it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   8%|▊         | 244/3000 [00:25<05:46,  7.96it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   8%|▊         | 248/3000 [00:25<03:46, 12.16it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   8%|▊         | 250/3000 [00:26<04:30, 10.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   8%|▊         | 252/3000 [00:26<04:08, 11.07it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   9%|▊         | 260/3000 [00:26<03:36, 12.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   9%|▊         | 262/3000 [00:27<04:10, 10.92it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   9%|▉         | 264/3000 [00:27<04:33, 10.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   9%|▉         | 267/3000 [00:27<05:18,  8.57it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   9%|▉         | 272/3000 [00:28<03:44, 12.14it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   9%|▉         | 274/3000 [00:28<03:34, 12.72it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   9%|▉         | 276/3000 [00:28<04:06, 11.04it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   9%|▉         | 280/3000 [00:28<04:08, 10.93it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   9%|▉         | 284/3000 [00:29<03:56, 11.48it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  10%|▉         | 286/3000 [00:29<05:16,  8.58it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  10%|▉         | 290/3000 [00:29<04:31,  9.96it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  10%|▉         | 295/3000 [00:30<03:22, 13.37it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  10%|▉         | 299/3000 [00:30<03:35, 12.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  10%|█         | 301/3000 [00:30<03:14, 13.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  10%|█         | 303/3000 [00:30<03:30, 12.84it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  10%|█         | 305/3000 [00:31<05:45,  7.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  10%|█         | 310/3000 [00:31<04:48,  9.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  10%|█         | 313/3000 [00:31<03:41, 12.11it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  11%|█         | 317/3000 [00:32<03:12, 13.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  11%|█         | 319/3000 [00:32<04:40,  9.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  11%|█         | 321/3000 [00:32<04:46,  9.35it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  11%|█         | 324/3000 [00:32<04:11, 10.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  11%|█         | 328/3000 [00:33<03:54, 11.40it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  11%|█         | 330/3000 [00:33<04:19, 10.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  11%|█         | 334/3000 [00:34<04:39,  9.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  11%|█▏        | 338/3000 [00:34<03:35, 12.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  11%|█▏        | 342/3000 [00:34<03:49, 11.60it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  12%|█▏        | 346/3000 [00:34<03:17, 13.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  12%|█▏        | 348/3000 [00:35<04:23, 10.05it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  12%|█▏        | 350/3000 [00:35<04:50,  9.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  12%|█▏        | 353/3000 [00:35<05:09,  8.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  12%|█▏        | 357/3000 [00:36<03:57, 11.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  12%|█▏        | 359/3000 [00:36<04:03, 10.83it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  12%|█▏        | 363/3000 [00:36<04:00, 10.97it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  12%|█▏        | 366/3000 [00:36<04:06, 10.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  12%|█▏        | 368/3000 [00:37<04:40,  9.38it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  12%|█▏        | 372/3000 [00:37<04:09, 10.55it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  13%|█▎        | 377/3000 [00:37<03:14, 13.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  13%|█▎        | 379/3000 [00:38<03:19, 13.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  13%|█▎        | 383/3000 [00:38<03:51, 11.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  13%|█▎        | 385/3000 [00:38<04:30,  9.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  13%|█▎        | 390/3000 [00:39<03:26, 12.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  13%|█▎        | 394/3000 [00:39<03:45, 11.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  13%|█▎        | 396/3000 [00:39<04:03, 10.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  13%|█▎        | 398/3000 [00:39<04:03, 10.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  13%|█▎        | 400/3000 [00:40<03:58, 10.89it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  14%|█▎        | 405/3000 [00:40<03:31, 12.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  14%|█▎        | 409/3000 [00:40<04:03, 10.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  14%|█▎        | 411/3000 [00:41<03:41, 11.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  14%|█▍        | 413/3000 [00:41<04:44,  9.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  14%|█▍        | 420/3000 [00:41<02:56, 14.63it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  14%|█▍        | 424/3000 [00:42<03:58, 10.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  14%|█▍        | 426/3000 [00:42<03:58, 10.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  14%|█▍        | 428/3000 [00:42<05:38,  7.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  14%|█▍        | 433/3000 [00:43<03:57, 10.80it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  15%|█▍        | 437/3000 [00:43<03:34, 11.96it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  15%|█▍        | 439/3000 [00:43<03:19, 12.83it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  15%|█▍        | 441/3000 [00:43<03:56, 10.84it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  15%|█▍        | 443/3000 [00:43<03:38, 11.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  15%|█▍        | 445/3000 [00:44<04:26,  9.60it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  15%|█▍        | 449/3000 [00:44<04:25,  9.60it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  15%|█▌        | 451/3000 [00:44<03:53, 10.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  15%|█▌        | 456/3000 [00:45<03:22, 12.55it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  15%|█▌        | 459/3000 [00:45<02:49, 15.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  15%|█▌        | 461/3000 [00:45<02:59, 14.14it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  15%|█▌        | 463/3000 [00:45<03:32, 11.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  16%|█▌        | 465/3000 [00:46<04:06, 10.29it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  16%|█▌        | 467/3000 [00:46<04:44,  8.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  16%|█▌        | 470/3000 [00:46<04:29,  9.40it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  16%|█▌        | 478/3000 [00:47<02:55, 14.40it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  16%|█▌        | 480/3000 [00:47<02:52, 14.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  16%|█▌        | 482/3000 [00:47<03:41, 11.35it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  16%|█▌        | 486/3000 [00:48<04:51,  8.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  16%|█▋        | 491/3000 [00:48<03:22, 12.42it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  16%|█▋        | 493/3000 [00:48<03:42, 11.28it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  17%|█▋        | 498/3000 [00:48<02:23, 17.47it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  17%|█▋        | 501/3000 [00:49<03:36, 11.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  17%|█▋        | 503/3000 [00:49<04:30,  9.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  17%|█▋        | 505/3000 [00:49<04:04, 10.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  17%|█▋        | 509/3000 [00:50<03:55, 10.58it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  17%|█▋        | 511/3000 [00:50<03:37, 11.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  17%|█▋        | 514/3000 [00:50<03:15, 12.74it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  17%|█▋        | 516/3000 [00:50<03:33, 11.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  17%|█▋        | 520/3000 [00:51<03:37, 11.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  18%|█▊        | 525/3000 [00:51<04:18,  9.58it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  18%|█▊        | 532/3000 [00:52<02:46, 14.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  18%|█▊        | 535/3000 [00:52<03:35, 11.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  18%|█▊        | 538/3000 [00:52<03:45, 10.93it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  18%|█▊        | 540/3000 [00:53<03:57, 10.35it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  18%|█▊        | 544/3000 [00:53<04:55,  8.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  18%|█▊        | 549/3000 [00:53<03:33, 11.48it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  18%|█▊        | 554/3000 [00:54<03:02, 13.38it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  19%|█▊        | 556/3000 [00:54<03:12, 12.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  19%|█▊        | 558/3000 [00:54<03:05, 13.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  19%|█▊        | 560/3000 [00:55<04:30,  9.04it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  19%|█▊        | 562/3000 [00:55<05:11,  7.82it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  19%|█▉        | 566/3000 [00:55<03:58, 10.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  19%|█▉        | 570/3000 [00:55<03:26, 11.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  19%|█▉        | 575/3000 [00:56<02:41, 15.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  19%|█▉        | 577/3000 [00:56<03:14, 12.48it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153056ebc3d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153042a90510>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fd48acd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fc09eed0>\n",
      "Evaluating workflow:  19%|█▉        | 579/3000 [00:57<05:58,  6.76it/s]Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530435b4750>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530435d3dd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530435d2d90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153042db5910>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153042db7bd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153042e2cbd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153042fa2210>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153042f68490>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153042f6ae90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153042f4e210>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056ee7bd0>, 1553829.090210887)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fce42dd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056d691d0>, 1553828.978025508)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fcdc70d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056ee5400>, 1553829.268323439)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fd489450>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056ee5390>, 1553829.155121194)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fc09c490>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056ee5a20>, 1553829.171658957)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153056e4f890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056ee6190>, 1553829.381499141)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153100fc3950>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056d683d0>, 1553829.192688204)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153042a90790>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056ee7310>, 1553829.301939774)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153042db57d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056ee4050>, 1553829.573125987)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153042db7a90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056ee4fa0>, 1553828.947258936)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153042e2d110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153042f14790>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530430c0810>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153043037fd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304301b8d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056d69320>, 1553829.37183265)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153056e14f50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056d69780>, 1553828.984064043)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530430c1010>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056d6b9a0>, 1553829.156985096)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153043037ed0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530432ae6d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056ee5e10>, 1553829.297506182)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153042e9e250>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303fc36550>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056ee5da0>, 1553828.669405867)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303fc36a50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303fcd2710>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303fce7850>, 1553830.878684733)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303fcd33d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303fb35c10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303fb467b0>, 1553831.294897605)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153042e2d0d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303fb577d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303fb7c6d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303fb40290>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303fb8f7d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303fba0690>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303fbb66d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303fb469e0>, 1553831.060280478)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303fba0550>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303fb45c50>, 1553831.078572295)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303fb7cd10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056ee78c0>, 1553831.11019572)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303fb40b10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303fb67f90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303fb45470>, 1553830.521960876)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303fb57a10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303fb465f0>, 1553831.194366265)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303fb8f690>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303fb47230>, 1553830.983597898)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303fbb5ed0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303fb45630>, 1553831.272410217)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303fb67e90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303fa554d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303fbe3540>, 1553832.879080897)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303fa55410>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530428b8a90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303fa981a0>, 1553833.099513829)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303fc76510>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303fc5e290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303fb44210>, 1553832.900720981)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153042f4e110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303fa33a10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303fa94a10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303fbe2b30>, 1553832.970060522)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303fa33390>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303fa98b40>, 1553832.883754)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303fa94890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303fa5fd90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303fa9a6d0>, 1553833.558096918)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303fa5f010>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303fb789d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303fa9aeb0>, 1553833.366573076)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303fb78c10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303faf7590>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303fa9b380>, 1553833.621699837)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303faf74d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303faa3ad0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303fbcd110>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303fa98de0>, 1553832.911819577)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303faa3990>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303fa998d0>, 1553832.802294106)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303fbcd650>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303f553810>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303f52ac10>, 1553834.526538344)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303f558090>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303f58bad0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303f52ae40>, 1553834.438334183)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303f58b950>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303fa95b50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303f52b310>, 1553835.036345408)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303fa95a10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303f587490>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303f5ccde0>, 1553835.032481529)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303f587410>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303f406c10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303f5cc600>, 1553836.357409783)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303f406b10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303f46c990>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303f474130>, 1553836.304771707)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303f46c790>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303f48d4d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303f529b00>, 1553836.248193761)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303f48d390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303f492550>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303f475080>, 1553836.468837716)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303f56e6d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303f49c410>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303f4a4110>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303f475240>, 1553836.445557708)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303f49c390>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303f475470>, 1553836.463182362)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303f4a4450>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303f47ca50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303f475c50>, 1553836.542076379)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303f4a6fd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303f4cea10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303f476740>, 1553836.812281918)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303f4cf050>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303f4d66d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303f2f6ed0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303f476cf0>, 1553836.680022223)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303f507310>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303f477230>, 1553837.19015138)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303f2f5890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303f4baf50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303f52bd90>, 1553836.954686242)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303f4bad10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303f5b1c90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303f30e1d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303f477a80>, 1553836.886184222)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303f5b2110>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303f477e00>, 1553837.025411828)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303f30e090>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303f334650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303f529710>, 1553837.035819238)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303f334450>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303f339190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056ee71c0>, 1553828.924703908)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153042fa2090>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056d6ac10>, 1553828.855669469)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153042fa1b90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056ee7770>, 1553828.804162877)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153042f6b590>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056ee60b0>, 1553829.29297701)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153042f4e0d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056ee6c80>, 1553829.481898727)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304301b790>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153207bc66d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303faf5610>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303fb0c850>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303fb11c90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303fb21a50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303fb33250>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530435d3850>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303fb63210>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303fbcfe10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303fbe6490>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303fbdcc50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303fa184d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303fbd6390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303fbaf410>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303fbcca10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303fa06410>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303fb632d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303fa75f10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303faab150>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303fac3850>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303face3d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303f4f04d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303f525c90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303f520750>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303f53ac50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303f564990>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303f56dad0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303fa1b090>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303f599d50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303f5ca590>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303f5e5e90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303f3fd710>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303f405410>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303f423a10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303f572cd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303f428e10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303f435750>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303f453dd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303f452c50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303fbe3150>, 1553835.982890302)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303f5df190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303f320a60>, 1553837.248295416)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303f3398d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303fcd5a20>, 1553830.647998427)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fd2b3810>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056ee4a60>, 1553830.28693596)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fe4056d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd221630>, 1553831.077160793)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303fb0cf10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303fcd5e10>, 1553830.44176768)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303fb135d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056d6add0>, 1553830.776268694)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303fb21850>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd221fd0>, 1553831.067721427)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303fb33050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd222740>, 1553831.073214405)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530435d31d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd222900>, 1553831.159231157)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303fb630d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303fb47a80>, 1553831.246446549)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303fbcfd10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303fce7770>, 1553831.950113176)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303fbe6b50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303fbe0c20>, 1553832.099620251)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303faf7950>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303fbe1b70>, 1553832.654437312)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303fa19750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303fbe2120>, 1553832.269065772)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303fbe6d90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303fb47070>, 1553831.573631634)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303fbafb50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056ee5240>, 1553831.445559524)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303fbcc850>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303fbe0130>, 1553832.460388156)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303fa05890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303fbe1390>, 1553833.130715976)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303fb612d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303fa98360>, 1553833.243881873)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303fa75d10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303fcd5cc0>, 1553833.315921655)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303faab010>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303fa9a200>, 1553833.470163771)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303fac3710>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303fb44600>, 1553833.275309815)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303face290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303fa9acf0>, 1553833.576946833)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303f4f22d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303f528440>, 1553834.102143379)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303f525a90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303fbe0ec0>, 1553834.121546952)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303f5207d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303fcd5d30>, 1553834.17790145)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303f53aa50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303f5287c0>, 1553834.713975968)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303f564790>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303f529c50>, 1553834.797425551)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303f56d8d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303f52aa50>, 1553834.533061884)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303fa1aa50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303f52b000>, 1553834.557708972)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303f599c10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056d699b0>, 1553835.415751729)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303f5ca390>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303f5cd470>, 1553835.196894474)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303f3fd510>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303f5cda90>, 1553835.073808424)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303f405550>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303f5ccd00>, 1553835.539179615)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303f4238d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303f52be70>, 1553835.045291567)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303f572c50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303f5ce430>, 1553835.015574216)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303fa3c9d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303f5cea50>, 1553835.974068892)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303f435e10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303fbe3c40>, 1553836.546195986)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303f525890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303f5cf230>, 1553835.979275036)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303f453310>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303f367cd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fcfeadd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fd3dba10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303fb44910>, 1553838.270578357)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153056c77d10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303f474e50>, 1553838.709170296)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fcfeb490>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303f474d70>, 1553838.112403868)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fd3d9990>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303f4d2490>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056cccc20>, 1553838.309089524)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303f4d2b50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303f356fd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303fb4fed0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056c53bd0>, 1553838.148861635)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303f357110>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056ee4130>, 1553838.336914812)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303f465e10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153042d0ea50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153042d2b050>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153042bb0150>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd221550>, 1553838.676589392)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303f584410>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd2235b0>, 1553838.688382118)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153042bb0610>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303f449a10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304bee8190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd5da350>, 1553838.235195585)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303f3484d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd5d8a60>, 1553838.50371463)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304bee8690>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304b1830d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303f474a60>, 1553838.682597188)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304b182e90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304885e890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd1d9080>, 1553838.644226243)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304885e690>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303f30ef10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd1d9cc0>, 1553838.421501163)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303f30edd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303f2f7890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303f476f90>, 1553839.171457001)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303f2f7790>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303f571d50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531051e6890>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd6b5b00>, 1553839.052469655)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531022d0c50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd6b4440>, 1553839.095762945)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531051e4dd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303f33a190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fc92abd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153106c1ff50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056e79320>, 1553838.388615817)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153042ebb090>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd6b6740>, 1553839.158633744)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303f339450>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd6b48a0>, 1553839.285057172)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fc928fd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303f4764a0>, 1553837.809008461)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153106c1e9d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fc3d9790>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fe436c80>, 1553840.148438523)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fc3d9510>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153100bdc510>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fe39fee0>, 1553840.554122569)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304beea510>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303f335750>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fcde1290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056c53b60>, 1553839.985007167)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303f4a6c90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056c50440>, 1553839.961523039)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fcde0f50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304bee9390>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fe39dc50>, 1553840.545230762)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15305784e990>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fc2de6d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fc4a6f50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056ee4590>, 1553841.387163687)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fc8f81d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fc8bb990>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fe39cec0>, 1553840.598023969)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fc8b9ad0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fcb32e90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fc9d8710>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153100bc34d0>, 1553841.008488408)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303f4a0fd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fc2eb790>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fc9a2f10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd346580>, 1553840.408566766)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fc2ebb50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd42cbb0>, 1553840.01789398)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fc9a3b10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304b182f50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153106ceb150>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd42f770>, 1553840.433345841)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304b190410>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd42f620>, 1553840.403739996)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153106ce9fd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fcaf5d10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153100bc3770>, 1553840.325346474)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fc4a56d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fe39cfa0>, 1553839.802602188)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fcb31790>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd42d780>, 1553840.702893261)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fc6e0b90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fc3e4a50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fcee09f0>, 1553841.148667353)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fc3e7610>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fc611650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd1da4a0>, 1553840.974650587)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fc610950>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fc69a550>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fcee3770>, 1553841.442176537)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fc698710>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fe535fd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd1d8590>, 1553841.937294794)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303f31c2d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fd69e250>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fccb1080>, 1553841.977747132)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fd69e4d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fc6898d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fccb1240>, 1553841.904549143)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fc689f50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fca2df50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fc69c750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd42e890>, 1553842.17702266)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fca2d850>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd758b40>, 1553842.078519792)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fc69d490>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fc5a5e90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd75b0e0>, 1553841.92045598)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fcce5b10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153107717550>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fe437070>, 1553842.978073543)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153107717cd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153106f54710>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fd524dd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153100bc10f0>, 1553842.269083805)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fc9dacd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153105246ac0>, 1553842.257641519)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fd524890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153042d61e10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fc711cd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fcb8f2d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fcee06e0>, 1553840.570039559)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fcb8d0d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153105244bb0>, 1553842.968036851)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303f32b450>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153105247a10>, 1553843.805404101)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fc710790>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153057703b50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fc8717d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153105244210>, 1553842.527173961)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153057701650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153100bc07c0>, 1553842.720450534)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fc873310>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fc4a54d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531079ba900>, 1553842.747714684)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fc4a66d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fce18f10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153057852d90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fcee2350>, 1553843.345270759)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fc3e7350>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fc7c9940>, 1553843.501014668)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fc9cadd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153106c9b6d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd42da90>, 1553842.854007567)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153106c98bd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fca80910>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fce4c750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056ccc910>, 1553843.466869881)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15310678ce90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fc7c9ef0>, 1553843.289223108)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fce4da50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fd4e6450>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fca3cc20>, 1553846.419410615)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fd4e6810>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fcba0fd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fc8cc6d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fca3e120>, 1553844.110034036)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fcba27d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fca3dc50>, 1553844.78949542)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fc8cd5d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fe3a5950>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fc7c87c0>, 1553843.783342885)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fe3a62d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153106f93590>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fe436c10>, 1553844.683223242)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303faf2850>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531069eb810>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fc6466d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fca8ddd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fc7c9630>, 1553842.389030157)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fca8ed10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fca3c4b0>, 1553844.000825898)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531069e8d90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd507700>, 1553844.378395466)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fc6473d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fe42da90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd504440>, 1553844.154336709)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fcecd7d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fc99b990>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153105244280>, 1553845.078525296)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fc99a1d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531077160d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531075c1c10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd13e190>, 1553844.543084611)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153107714fd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd13d7f0>, 1553844.82142219)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fc5a84d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fc99b5d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fcd0a590>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd13dfd0>, 1553844.944487483)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fc999610>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd13fe00>, 1553844.92933532)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fcd0b490>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fcbe15d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd13f310>, 1553844.519202736)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fc9a2490>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fd448f90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057e918d0>, 1553845.184737517)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fd448550>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fca53510>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd13de80>, 1553845.318138997)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fd341350>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fe3a5790>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fc539050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fc7cab30>, 1553845.355502499)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fe3a7390>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057e924a0>, 1553845.723609504)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fc56add0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fc95cc10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153101bf6910>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057e91b70>, 1553845.611113026)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153101bf4510>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056c538c0>, 1553845.757650625)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530578a60d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153106b92ad0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057e91390>, 1553845.937205706)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fc95d350>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd5062e0>, 1553844.177490831)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531075fcc10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153057ac2450>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057f42b30>, 1553846.131261693)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153106bc0590>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15310731bcd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530579c0490>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057f414e0>, 1553846.039940854)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15310731a750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd6b7620>, 1553846.174950876)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530579c0e50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fc37a790>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153057af6690>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057f42900>, 1553846.343934454)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153057af61d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153106f91190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd758210>, 1553847.067605308)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153106f93d90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153057a384d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057fde200>, 1553847.079972662)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153057a38fd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530578f8410>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153105244b40>, 1553847.084230386)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fd417b50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fc998790>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fc447090>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057fdef20>, 1553846.357586001)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fc999750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057fdd630>, 1553846.841352584)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fc446b10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fc29ebd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fd381ad0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057fdda20>, 1553847.41276699)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fc29ead0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057fdef90>, 1553846.929769094)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153105f663d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153057874690>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fc520d70>, 1553847.661339385)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fc79f590>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153057a35290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fc521710>, 1553847.835279264)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153057a366d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fc290190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531052477e0>, 1553846.334563808)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fc37a510>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303f5cd010>, 1553847.317349799)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531063f6790>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15310726f990>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15310779f4d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fccb37e0>, 1553847.594289363)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15310726f290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303f5cec10>, 1553847.786768748)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15310779de50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fd10cf50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303f5cc520>, 1553847.452623076)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fd10d510>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153107300d90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057f41400>, 1553847.900683035)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153203649610>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fc98d410>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fe39d550>, 1553847.901747274)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fc98e5d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530578f4550>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fc07bc50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303fa98980>, 1553848.065336182)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530578f5210>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303f320670>, 1553848.79904762)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fc079d50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fc1acbd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153057949790>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fc521e10>, 1553848.89407437)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fc1ad210>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303f320600>, 1553848.932370731)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153057949a50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153105d6e290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303f320830>, 1553848.711041551)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153105d6dd10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fc1398d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057fde7b0>, 1553848.955062992)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fc139e10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153057aa2fd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303f322190>, 1553848.830418287)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153057aa27d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fd219650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303f322c10>, 1553849.21358917)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fd381310>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fc446190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15310114ec90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fd484790>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303f5cc6e0>, 1553847.296549559)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fd4842d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303f322e40>, 1553848.977800961)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fc4458d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303f323000>, 1553849.683601814)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15310114f750>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15305666be90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b7c01a0>, 1553849.395479046)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15305666bd50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fc67ee10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b7c04b0>, 1553849.452903198)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fc198850>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531052756d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b7c08a0>, 1553849.095206317)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153105274f90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fd5304d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057fdfe70>, 1553849.665900354)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fd532d90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fd370e10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153105b44b50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fc520e50>, 1553849.601469196)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15310114f610>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fca3fd90>, 1553849.651237049)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153105b473d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fce6a890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b7c2270>, 1553849.935863679)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fce6ab10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531023f9b10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153106e6ae50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b7c26d0>, 1553850.215442328)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531023fb110>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b7c2890>, 1553850.046285707)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153106e68ad0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153105e41b90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b7c2cf0>, 1553850.195512298)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153105e40090>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fd212e10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b7c32a0>, 1553850.392468607)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fd212950>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153057948ed0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303f3235b0>, 1553850.676416168)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fc10d890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fd658ad0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd13f930>, 1553848.950168051)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fd65ad90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303f5ce040>, 1553850.596468772)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153057949ed0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fc9a5310>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531050f9160>, 1553850.971545296)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fc9a5890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fd10d410>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153100c10a50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531050f96a0>, 1553851.471367818)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fccce890>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531050f9b70>, 1553850.764893817)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153100c131d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fc4cdd50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531076c4090>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531050f9d30>, 1553850.916739751)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fc4cc310>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531050f9f60>, 1553850.857516898)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531076c5090>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fcf9d150>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531050faba0>, 1553851.024339285)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fcf9f6d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fe3b8c10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057e92dd0>, 1553851.563096173)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fe3bbb50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153056cf7b10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530577254d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531050fa350>, 1553851.400051487)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153056cf7290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531050fbcb0>, 1553852.081559203)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153057725f10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fc851510>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fd7658d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153105150590>, 1553851.169337932)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fc850510>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531051507c0>, 1553851.478051004)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fd766c50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153056cc77d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531051509f0>, 1553851.640755417)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153056cc6c50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fd3b66d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531051515c0>, 1553851.717778473)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fd11add0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fd148a10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b7c1e80>, 1553851.68388024)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fc538890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fd020b50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153056c73410>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fd3a8510>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303f323c40>, 1553850.472291581)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fd3a96d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057fdec10>, 1553852.077289382)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fd022650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531051522e0>, 1553852.03730136)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153056c71390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153056c7d4d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531051533f0>, 1553852.19705705)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153056c7e450>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153056de0c90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153105153770>, 1553852.70568729)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153056de3650>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153057869d50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153056e42450>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153105153af0>, 1553852.505156255)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153057869390>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531051ac3d0>, 1553852.655126649)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153056e43c90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153056d7cc90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153056d12190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fca3fa80>, 1553853.203014635)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153106d62dd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153056d12750>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153056d84e10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531051ad080>, 1553852.739199914)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153056d122d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531051ad5c0>, 1553852.651073554)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fd004cd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153056c3b810>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fca3ec80>, 1553852.948222105)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153056c39510>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153106e68350>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153056d404d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b7c2ba0>, 1553853.27760194)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15310230b850>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531051adb70>, 1553853.205361971)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153056d43b50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153056efb190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531051af000>, 1553853.084253294)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153056efa810>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153056c3c390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153056dc6510>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153056c97850>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531051518d0>, 1553852.771720462)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153056d7f210>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153105152dd0>, 1553851.791370223)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153056c94810>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531051af690>, 1553853.06927064)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15310709e310>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531051afaf0>, 1553853.24967385)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153056dc40d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153056c57410>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b984130>, 1553853.63384075)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153056c57090>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153056d26c90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531051af460>, 1553853.717903755)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153056d27ad0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153056c0cc90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b985390>, 1553854.074589531)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153056c0d7d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153056c72f90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fe519d10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531050fbaf0>, 1553853.803555243)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fc4c4e10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b985940>, 1553853.957308687)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fe5188d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153056eaa410>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b9863c0>, 1553853.716742277)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153056ea8bd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153056d8a610>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153056e19c10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b986cf0>, 1553854.330530556)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153056de7150>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b9873f0>, 1553854.095547601)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153056d13790>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153056d89890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b9875b0>, 1553854.779619971)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153056d89790>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fce30b90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fd0e8d50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531050f9860>, 1553854.869503024)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153056e11ad0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153105151be0>, 1553854.504254122)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fd0e9a50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153056d77350>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303fa9be70>, 1553855.02816719)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153056d77590>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153056e7c750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bd10980>, 1553854.662783288)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15310526eb90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153056de84d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b984050>, 1553855.039074275)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153056e7e090>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b984360>, 1553853.263378651)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153056de8b90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153056dc7f50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530579f9510>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bd11a20>, 1553855.17935401)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153056dc6810>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fc701fd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fd51b850>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bd127b0>, 1553854.840393024)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fc700b90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bd129e0>, 1553855.247641803)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fd51aa10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fcfb7790>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bd12ba0>, 1553855.419623338)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fcfb5090>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fc2ed3d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531070ecd90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fcb08550>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bd13ee0>, 1553855.839149153)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fd2c2610>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bd8c130>, 1553855.846814044)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fcb09150>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531077bbb10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fd282b90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b7c0c20>, 1553855.649396317)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153107781790>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bd8d080>, 1553855.804273488)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fd2829d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fc2e01d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bd8d320>, 1553855.826951409)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153101b5cbd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fcf34850>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fc7d0b50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bd8d630>, 1553855.653065648)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153105cf7f90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b987a10>, 1553856.132235081)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fca96550>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153106268610>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b7c3070>, 1553856.551656799)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153106268f10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bd11ef0>, 1553855.062145632)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530579fb590>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530579f8950>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531051530e0>, 1553855.698177523)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fc2ed890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bd120b0>, 1553854.892221624)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fcc0d3d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153056d7ff50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153057a466d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bd8f000>, 1553856.546197115)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153056d7e1d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153056ee3510>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fd6886d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bc28360>, 1553856.889791816)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fd6891d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153105d50650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bc28600>, 1553857.17266976)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fca60e50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153105262510>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fd1de6d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bd11e10>, 1553857.043543773)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531052635d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b987e70>, 1553857.127448761)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fd1de890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153056c60610>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531051aff50>, 1553856.918488593)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fd4a5590>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fcab0bd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fcf1c5d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bc297f0>, 1553856.769443175)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fcab1690>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b7c1d30>, 1553857.231258921)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fcf1ec50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fe52d4d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bc2a890>, 1553857.111903844)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fd2f3f10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fd281c90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fc7373d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bc29ef0>, 1553857.336727236)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153105b06450>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bc29e10>, 1553857.582912591)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fc736e90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fc460ed0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bd13150>, 1553857.616429518)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fc460610>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fe495e50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153106d27fd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bd8fa10>, 1553856.342309104)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fd3dd6d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bd8eba0>, 1553856.803386997)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153056ee3ad0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bd13460>, 1553857.512773009)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fe495590>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b986190>, 1553857.801874759)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153106d246d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fe4815d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153105256750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bc2af20>, 1553857.366584941)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153106b52850>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bca8830>, 1553857.772812214)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531052544d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fd0a34d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bca92b0>, 1553858.110092268)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fd0a1ad0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fe3ae290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bca99b0>, 1553858.346017895)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fe3ad010>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153106ccb890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bd8dc50>, 1553858.305699285)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153106cc9c90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fc58df10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bcaa270>, 1553858.391493675)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fc0dad50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15310638eb50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fcf9b750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bc28520>, 1553858.979704208)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15310638ead0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153056e99b10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bcabd90>, 1553858.833610921)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153056e98810>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531062f2f10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bb20050>, 1553858.650097558)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531062f3210>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fc467b90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bb20360>, 1553858.89585048)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531021c7d10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fd01d010>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fc09ff10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bb20f30>, 1553858.776270391)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fc465fd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bb210f0>, 1553859.171475189)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fc09c450>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fd228350>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bb21780>, 1553858.935056542)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fd28a190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fcd9d150>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bd12580>, 1553858.931436042)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fcf98c50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bd12430>, 1553858.704320998)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fe52f4d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fc737f90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fcc01b90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bb22430>, 1553858.822067958)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fc735d90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bb225f0>, 1553859.813868198)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fcc00b90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fd4ceed0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bc2a200>, 1553859.554586699)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fd4cec90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fd61e890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bb23460>, 1553859.609491758)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fd61e510>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fd53b490>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bb23a80>, 1553859.782625773)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fd539c50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153057be4e50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bb23b60>, 1553859.852799062)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153057be5910>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530579e77d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153056d05f50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bba46e0>, 1553860.195893589)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530579e4110>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bba4c90>, 1553859.989838106)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153056ebe6d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153056d8e9d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bba50f0>, 1553860.25906941)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153056d8cd90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531062f0510>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bba5400>, 1553860.132007097)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531062f1b10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530579d3950>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bc28f30>, 1553860.499675832)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530579d06d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153056d09d50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304ba12c10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bba4fa0>, 1553860.323400473)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153056d08950>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bcab000>, 1553860.567942856)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fc2ec550>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304ba1c810>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304ba38150>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fd12c990>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bb21470>, 1553861.163433846)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153056c04190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bba75b0>, 1553860.656902334)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304ba387d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bc2b7e0>, 1553859.121603036)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fd12fd90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304ba43e50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bba6040>, 1553861.33395595)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304ba43d90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304ba55290>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304ba67890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304ba60440>, 1553861.191204977)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304ba56e50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304ba60670>, 1553861.161692545)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304ba67690>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304ba6e250>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304ba879d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304ba60830>, 1553860.942659094)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304ba6e110>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bc2a580>, 1553860.876411587)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304ba87650>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304baa0750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304ba61240>, 1553861.04546982)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304baa0310>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304baba790>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304ba62270>, 1553861.490216357)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304bac6d10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304bae9510>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304baeb690>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304ba61710>, 1553861.60728436)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304baeb550>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304baf5710>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304b805d50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304ba62a50>, 1553861.774112297)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304baf5950>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bd8e900>, 1553861.910305791)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304b805b50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304b840750>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304b849b10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b8541a0>, 1553861.805320183)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304b849910>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304b80a250>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304b871e50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b854590>, 1553861.876189016)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fcdd54d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304b885e90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bb22120>, 1553861.275809637)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304bae9310>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bd8f3f0>, 1553861.650168639)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fc1f8f10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304ba637e0>, 1553861.911475701)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304b871d10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b855160>, 1553862.228068052)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304ba85f10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304b8954d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bba6ba0>, 1553862.29829459)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304b896150>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304b8a0650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b8556a0>, 1553863.335507833)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304b8a0510>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304b807610>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bba6190>, 1553862.982599719)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153105fa5b50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304ba678d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bba7a10>, 1553862.630787058)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304ba41810>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304ba552d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304b61e6d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b855fd0>, 1553862.494746398)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304ba57f50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bd8cd70>, 1553862.878511345)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304b61fc90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304b63d490>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b658050>, 1553863.303464038)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304b63d7d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304b8f2610>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304b646590>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b857e00>, 1553863.000744823)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304b8f1f90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b658280>, 1553862.933011137)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304b646450>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304b67a2d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b658c90>, 1553862.75730347)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304b67a850>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304b60b990>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304ba602f0>, 1553863.139581204)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304bad04d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304b69a290>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304b6b3250>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b855cc0>, 1553863.744272324)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304b69a110>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b659ef0>, 1553863.585240038)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304b6b3110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304b6cbad0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b6594e0>, 1553863.737941156)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304b6cbf50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304b6f50d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bba55c0>, 1553863.524377654)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304b6f4f90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304b6aa5d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b65aac0>, 1553863.482086751)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304b6aa110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304b601e50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304b534cd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b65b4d0>, 1553863.988951323)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304b698410>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b65b690>, 1553864.27078046)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304b534b90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304b551590>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b8544b0>, 1553864.326289016)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304b551410>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304b566e10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b558600>, 1553864.909898552)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304b566cd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304b583990>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b558d70>, 1553864.100566939)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304b583e90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304b620f50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b559860>, 1553864.859542522)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304b870c50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304b59f510>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b559a20>, 1553864.520060113)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304b59ee90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304b647490>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304b5e1bd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b55a190>, 1553864.794145266)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304b58add0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304ba60b40>, 1553864.577172372)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304b5e1a90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304b5f4110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304b408a90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304b432210>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b856120>, 1553864.709969327)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304b432010>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304b44b050>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304b41cc90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304b518610>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b856820>, 1553864.638738855)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304b5f4250>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b55af90>, 1553864.625363488)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304b4091d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bd8f700>, 1553865.072133678)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304b44af10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b45c440>, 1553865.499454975)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304b690990>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b8548a0>, 1553863.698643894)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304b50bf50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304b4943d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304ba61860>, 1553865.992274771)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304b494a90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304b495710>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b45d010>, 1553865.367499204)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304b495fd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304b4310d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304b4c8dd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b45d320>, 1553865.189174094)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304b4a0250>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b55b310>, 1553865.506044169)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304b4c9510>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304b4cb950>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b45dd30>, 1553865.065299631)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304b4cb750>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304b4d97d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b45df60>, 1553865.784635804)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304b4d9a90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304b4ea850>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304b315150>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b45e900>, 1553866.091455208)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304b4eab10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b45eeb0>, 1553865.5720712)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304b314f50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304b32bd10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bba6b30>, 1553865.979813059)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304b32bbd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304b535ed0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304ba60ad0>, 1553866.083616186)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304b5357d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304b361650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b559240>, 1553866.155115032)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304b5af190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304b5e3810>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b358ec0>, 1553866.38453068)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304b5655d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fd51b890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304b3b4310>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b359320>, 1553866.114546924)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304b454410>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b3594e0>, 1553868.429136926)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304b3b4950>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304b3ba4d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304ba60e50>, 1553866.289968742)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304b3ba2d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304b36e6d0>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304b3f0490>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b35a040>, 1553867.073935657)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304b36df50>\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b35a5f0>, 1553866.819547133)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304b3f0bd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304b4d8d10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b35a900>, 1553866.488770484)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304b4d8dd0>\n",
      "Evaluating workflow:  19%|█▉        | 584/3000 [00:57<04:15,  9.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  20%|█▉        | 591/3000 [00:57<02:30, 16.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  20%|█▉        | 594/3000 [00:57<02:21, 17.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  20%|█▉        | 597/3000 [00:58<02:47, 14.32it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  20%|██        | 602/3000 [00:59<04:20,  9.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  20%|██        | 604/3000 [00:59<03:59, 10.02it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  20%|██        | 609/3000 [00:59<03:19, 12.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  20%|██        | 612/3000 [00:59<02:47, 14.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  20%|██        | 615/3000 [00:59<02:46, 14.35it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  21%|██        | 620/3000 [01:00<03:45, 10.57it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  21%|██        | 622/3000 [01:00<04:33,  8.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  21%|██        | 627/3000 [01:01<03:21, 11.80it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  21%|██        | 632/3000 [01:01<02:34, 15.36it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  21%|██        | 634/3000 [01:01<02:31, 15.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  21%|██        | 636/3000 [01:01<03:50, 10.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  21%|██▏       | 638/3000 [01:02<04:26,  8.87it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  21%|██▏       | 643/3000 [01:02<03:35, 10.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  22%|██▏       | 645/3000 [01:02<03:26, 11.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  22%|██▏       | 650/3000 [01:02<02:21, 16.57it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  22%|██▏       | 652/3000 [01:03<03:16, 11.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  22%|██▏       | 654/3000 [01:03<04:07,  9.49it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  22%|██▏       | 656/3000 [01:03<03:36, 10.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  22%|██▏       | 660/3000 [01:04<03:41, 10.55it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  22%|██▏       | 666/3000 [01:04<03:14, 11.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  22%|██▏       | 668/3000 [01:04<03:15, 11.93it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  22%|██▏       | 671/3000 [01:05<03:17, 11.82it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  22%|██▏       | 673/3000 [01:05<04:08,  9.36it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  22%|██▎       | 675/3000 [01:05<04:35,  8.42it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  23%|██▎       | 678/3000 [01:05<04:01,  9.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  23%|██▎       | 682/3000 [01:06<03:47, 10.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  23%|██▎       | 686/3000 [01:06<02:44, 14.06it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  23%|██▎       | 690/3000 [01:07<03:28, 11.09it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  23%|██▎       | 694/3000 [01:07<02:37, 14.60it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  23%|██▎       | 696/3000 [01:07<03:50,  9.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  23%|██▎       | 700/3000 [01:08<04:06,  9.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  23%|██▎       | 702/3000 [01:08<04:11,  9.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  24%|██▎       | 707/3000 [01:08<02:52, 13.28it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  24%|██▎       | 709/3000 [01:08<03:06, 12.27it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  24%|██▍       | 715/3000 [01:09<02:21, 16.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  24%|██▍       | 717/3000 [01:09<03:54,  9.72it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  24%|██▍       | 719/3000 [01:09<03:59,  9.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  24%|██▍       | 723/3000 [01:10<03:32, 10.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  24%|██▍       | 725/3000 [01:10<03:58,  9.55it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  24%|██▍       | 727/3000 [01:10<03:30, 10.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  24%|██▍       | 734/3000 [01:10<03:08, 12.02it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  25%|██▍       | 738/3000 [01:11<04:03,  9.29it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  25%|██▍       | 742/3000 [01:11<03:18, 11.40it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  25%|██▍       | 744/3000 [01:12<03:34, 10.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  25%|██▍       | 746/3000 [01:12<03:45, 10.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  25%|██▍       | 749/3000 [01:12<02:50, 13.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  25%|██▌       | 752/3000 [01:12<03:01, 12.42it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  25%|██▌       | 757/3000 [01:13<02:56, 12.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  25%|██▌       | 761/3000 [01:13<03:22, 11.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  25%|██▌       | 763/3000 [01:13<03:38, 10.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  26%|██▌       | 765/3000 [01:13<03:23, 10.96it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  26%|██▌       | 769/3000 [01:14<03:31, 10.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  26%|██▌       | 771/3000 [01:14<03:33, 10.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  26%|██▌       | 776/3000 [01:14<02:34, 14.38it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  26%|██▌       | 778/3000 [01:15<03:07, 11.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  26%|██▌       | 782/3000 [01:15<03:27, 10.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  26%|██▌       | 786/3000 [01:15<03:02, 12.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  26%|██▋       | 793/3000 [01:16<02:37, 14.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  26%|██▋       | 795/3000 [01:16<02:27, 14.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  27%|██▋       | 797/3000 [01:16<03:05, 11.87it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  27%|██▋       | 799/3000 [01:17<04:54,  7.46it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  27%|██▋       | 804/3000 [01:17<03:40,  9.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  27%|██▋       | 807/3000 [01:17<03:08, 11.61it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  27%|██▋       | 811/3000 [01:18<03:17, 11.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  27%|██▋       | 814/3000 [01:18<03:06, 11.72it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  27%|██▋       | 816/3000 [01:18<03:15, 11.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  27%|██▋       | 818/3000 [01:18<02:58, 12.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  27%|██▋       | 820/3000 [01:18<03:26, 10.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  27%|██▋       | 824/3000 [01:19<03:07, 11.60it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  28%|██▊       | 828/3000 [01:19<03:48,  9.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  28%|██▊       | 830/3000 [01:19<03:19, 10.87it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  28%|██▊       | 832/3000 [01:20<03:33, 10.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  28%|██▊       | 837/3000 [01:20<03:00, 11.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  28%|██▊       | 839/3000 [01:20<02:43, 13.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  28%|██▊       | 844/3000 [01:21<02:50, 12.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  28%|██▊       | 848/3000 [01:21<04:01,  8.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  28%|██▊       | 852/3000 [01:21<02:50, 12.63it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  29%|██▊       | 856/3000 [01:22<02:32, 14.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  29%|██▊       | 860/3000 [01:22<02:53, 12.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  29%|██▉       | 863/3000 [01:22<02:33, 13.88it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  29%|██▉       | 865/3000 [01:22<03:12, 11.11it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  29%|██▉       | 869/3000 [01:23<03:09, 11.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  29%|██▉       | 871/3000 [01:23<02:49, 12.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  29%|██▉       | 873/3000 [01:23<03:33,  9.97it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  29%|██▉       | 875/3000 [01:23<03:27, 10.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  29%|██▉       | 883/3000 [01:24<02:36, 13.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  30%|██▉       | 887/3000 [01:24<03:02, 11.55it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  30%|██▉       | 889/3000 [01:25<02:56, 11.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  30%|██▉       | 891/3000 [01:25<02:54, 12.11it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  30%|██▉       | 893/3000 [01:25<03:48,  9.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  30%|██▉       | 899/3000 [01:25<02:45, 12.66it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  30%|███       | 903/3000 [01:26<02:47, 12.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  30%|███       | 905/3000 [01:26<03:29,  9.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  30%|███       | 909/3000 [01:26<03:00, 11.57it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  30%|███       | 911/3000 [01:27<03:18, 10.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  30%|███       | 913/3000 [01:27<03:27, 10.04it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  31%|███       | 919/3000 [01:27<02:49, 12.27it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  31%|███       | 921/3000 [01:28<03:13, 10.72it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  31%|███       | 926/3000 [01:28<02:56, 11.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  31%|███       | 930/3000 [01:28<02:56, 11.72it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  31%|███       | 932/3000 [01:29<04:08,  8.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  31%|███       | 936/3000 [01:29<03:53,  8.83it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  31%|███▏      | 940/3000 [01:29<03:20, 10.28it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  31%|███▏      | 944/3000 [01:30<02:25, 14.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  32%|███▏      | 948/3000 [01:30<02:36, 13.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  32%|███▏      | 951/3000 [01:30<02:27, 13.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  32%|███▏      | 953/3000 [01:31<03:39,  9.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  32%|███▏      | 957/3000 [01:31<03:59,  8.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  32%|███▏      | 963/3000 [01:32<02:47, 12.16it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  32%|███▏      | 965/3000 [01:32<02:54, 11.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  32%|███▏      | 967/3000 [01:32<03:18, 10.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  32%|███▏      | 971/3000 [01:32<02:47, 12.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  32%|███▏      | 974/3000 [01:32<02:35, 12.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  33%|███▎      | 980/3000 [01:33<02:44, 12.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  33%|███▎      | 982/3000 [01:33<02:34, 13.05it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  33%|███▎      | 984/3000 [01:33<02:59, 11.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  33%|███▎      | 988/3000 [01:34<02:59, 11.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  33%|███▎      | 990/3000 [01:34<03:20, 10.02it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  33%|███▎      | 992/3000 [01:34<03:01, 11.07it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  33%|███▎      | 994/3000 [01:34<03:27,  9.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  33%|███▎      | 999/3000 [01:35<02:49, 11.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  33%|███▎      | 1001/3000 [01:35<02:35, 12.89it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  33%|███▎      | 1003/3000 [01:35<02:40, 12.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  34%|███▎      | 1005/3000 [01:35<03:17, 10.11it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  34%|███▎      | 1009/3000 [01:36<03:04, 10.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  34%|███▎      | 1011/3000 [01:36<03:30,  9.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  34%|███▍      | 1014/3000 [01:36<03:25,  9.67it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  34%|███▍      | 1019/3000 [01:37<02:31, 13.04it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  34%|███▍      | 1021/3000 [01:37<03:15, 10.14it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  34%|███▍      | 1026/3000 [01:37<03:07, 10.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  34%|███▍      | 1028/3000 [01:38<03:10, 10.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  34%|███▍      | 1030/3000 [01:38<03:20,  9.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  35%|███▍      | 1036/3000 [01:38<02:46, 11.77it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  35%|███▍      | 1039/3000 [01:39<02:40, 12.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  35%|███▍      | 1041/3000 [01:39<02:26, 13.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  35%|███▍      | 1046/3000 [01:39<02:17, 14.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  35%|███▍      | 1048/3000 [01:39<02:53, 11.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  35%|███▌      | 1050/3000 [01:40<04:29,  7.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  35%|███▌      | 1052/3000 [01:40<04:04,  7.97it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  35%|███▌      | 1059/3000 [01:40<02:29, 13.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  35%|███▌      | 1063/3000 [01:41<02:28, 13.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  36%|███▌      | 1065/3000 [01:41<02:54, 11.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  36%|███▌      | 1067/3000 [01:41<02:44, 11.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  36%|███▌      | 1069/3000 [01:42<03:32,  9.07it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  36%|███▌      | 1071/3000 [01:42<04:12,  7.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  36%|███▌      | 1075/3000 [01:42<03:12,  9.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  36%|███▌      | 1080/3000 [01:43<02:36, 12.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  36%|███▌      | 1082/3000 [01:43<03:12,  9.97it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  36%|███▌      | 1086/3000 [01:43<02:37, 12.17it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  36%|███▋      | 1091/3000 [01:44<03:09, 10.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  36%|███▋      | 1093/3000 [01:44<03:01, 10.52it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  37%|███▋      | 1097/3000 [01:44<02:52, 11.02it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  37%|███▋      | 1101/3000 [01:45<03:01, 10.47it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  37%|███▋      | 1103/3000 [01:45<02:49, 11.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  37%|███▋      | 1107/3000 [01:45<02:16, 13.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  37%|███▋      | 1109/3000 [01:46<04:00,  7.87it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  37%|███▋      | 1117/3000 [01:46<02:22, 13.17it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304b3a5850>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b35b380>, 1553866.566880721)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304b3a4750>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304b273f90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b260c90>, 1553867.381305742)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304b307fd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304b2cf210>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b3582f0>, 1553867.797375921)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304be04ad0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304be140d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304be3f9d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b2638c0>, 1553867.901749737)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304be14810>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b558440>, 1553867.75826942)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304be3f890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153056694650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15305667a740>, 1553869.94379086)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530566ed190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530566eb990>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b65a120>, 1553870.517979687)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304886ec90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304887b350>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530488b9550>, 1553870.216408409)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304887b810>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304beef690>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530566789f0>, 1553869.349165707)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304beef2d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153056621010>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530488bb1c0>, 1553871.328479604)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153056621ad0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304896f410>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048975400>, 1553872.14589506)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304896edd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530566209d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530488bb460>, 1553872.348263024)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304bf3b2d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530489ead10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bf1dc50>, 1553872.565601571)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304bf6fe90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304bf204d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304bf22ed0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bf1c4b0>, 1553872.102044692)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304bf23210>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304be48e50>, 1553872.17303018)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304bf20dd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153048a88b10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048b242f0>, 1553874.324560459)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153048a89b90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153104fe1310>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048b26820>, 1553875.541578304)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153104fe10d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153048ac7390>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153105035a90>, 1553875.822432562)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531050573d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304bf20f90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153105036430>, 1553875.809323257)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304bf22110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153104faf3d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153105036f20>, 1553876.356775697)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153105056e10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153048abf290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153104f14590>, 1553876.727579591)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153104f8fa50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153104ec6990>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153104f17d90>, 1553877.44760772)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153104fd1850>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531044effd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531044e9cc0>, 1553878.136433665)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531044efc10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531050565d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531044ea3c0>, 1553877.813372927)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304bfbf5d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531043872d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153105037ee0>, 1553878.933175161)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531043ba810>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531043a5990>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531043d16a0>, 1553878.944428271)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153104463b90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153104433310>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531043d2cf0>, 1553879.052872816)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153103cb5590>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531043f2ed0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531044e9320>, 1553878.856711153)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531043f30d0>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153103cdb5d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048a4a820>, 1553879.545385166)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153103cdb210>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15310454bf10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153103d05080>, 1553879.54014629)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153104360310>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153103bfe3d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15310453e750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531043d3f50>, 1553879.12479287)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15310453ead0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153103c00d70>, 1553881.671503446)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153103bfe690>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153104400350>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153103c01b70>, 1553881.298604889)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153103c35490>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153103c2ae90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153103c4de50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153103c71050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153103c01d30>, 1553881.230533004)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153103c4dd10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153103c01630>, 1553880.767258878)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153103c2b3d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153103c02580>, 1553880.899798369)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153103c70f10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153103948c50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153103c02270>, 1553883.228238363)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153103d3b3d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531037c7f50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153103d06cf0>, 1553882.422730671)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531037c75d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153103833810>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531036dad10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531038d58d0>, 1553884.663669874)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153103833cd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15310389d320>, 1553884.459657135)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531036db250>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153103911a90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153103d066d0>, 1553884.45347323)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531039119d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531036f2fd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531036fa390>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15310389d940>, 1553884.489476644)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531036f2f50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531038d7d90>, 1553884.476459392)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531036fa190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153103717dd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15310389e6d0>, 1553884.634451548)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153103952dd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15310373f110>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15310389d710>, 1553885.026694083)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15310373ef10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153103764510>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15310389e120>, 1553884.939596584)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153103764310>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531037bfa90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15310377a0d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15310389f9a0>, 1553884.771399132)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531037be190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531037c15c0>, 1553884.572121567)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15310377a610>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153103814610>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15310389d780>, 1553884.651189465)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15310378ff90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15310377bad0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531037a2750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153103784d70>, 1553884.91748433)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531036bba50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153103784bb0>, 1553884.788379521)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531037a2450>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531035b3f10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153103784fa0>, 1553884.681684504)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531035b3dd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153103999d90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531035e2dd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153103785710>, 1553885.225305448)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153103999e10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153103785c50>, 1553884.802978024)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531035e0710>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15310377ea90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304b2154d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304b250d50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304b271f10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304b2a84d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304b2aa690>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304beee6d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304b2708d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304b2c7250>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304b2fa250>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304befea10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304be63710>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304be942d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304be566d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304b433350>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530566011d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153056616610>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153056638350>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304bea1f90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15305665c2d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153056621810>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153056684410>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304b2ce210>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304b3b7410>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530566b4990>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530566ca990>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530566ab410>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15305664c510>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153048888d90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304889b910>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530488d2950>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530488ec450>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304888bd10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304886ce50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530488c5990>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153048930350>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153048913490>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153048954050>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153048956650>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153048991490>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530489b07d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530489c22d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530489d6350>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15305664e410>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153048a13cd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153048a32490>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153048944ad0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153048a1e110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304bf004d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304bf61750>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304bf6de10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304bf90cd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304bf9fb50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304bfbc750>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304bf39810>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304bfef790>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153048956450>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153048a51110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153048a6b510>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153048a80610>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153048aa50d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153048ab3410>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153048a36d50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153048ad20d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153048af1a90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304bf39ed0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153048b09dd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153048b1c990>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153104f6f650>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304bfbd590>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153048abf250>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153104fc4350>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153104f67190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153104fb4210>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153105000390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153105023ed0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153104e69690>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153104e87410>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153105039f10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153104eb0fd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153104ec5f50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153104ee3f90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153104eef590>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153105020fd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304bf915d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153104f59510>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153104460910>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153104fc4590>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15310447ec50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153104491c50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153104e87750>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531044b68d0>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531044edc50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15310450d8d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304bf384d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153104523510>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153104ec6710>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15310453d910>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15310454bf50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153104393fd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531043c6210>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531043e8b10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153104412650>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15310442e6d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531044cec50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153103cc18d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531043b9d50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531044504d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153103d0a5d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153103d30b10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153103d437d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153103d630d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153103d6d750>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153103bbc250>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153103cc2b10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153103bd45d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153103bf4ad0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153103bf6a50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153103c201d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153103c7f550>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153103ca1150>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531038c2890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531038efd90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531038d2390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153103910dd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153103bcd4d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153103cdaa50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531038ea450>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153103951510>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153103c2bf50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15310396f490>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15310397ccd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531039736d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531037be6d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531043a9890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531038b74d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531037fd4d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153103821850>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15310382b490>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531038cf510>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153103850150>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153103bfc750>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531038804d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15310386a910>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153103897550>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153103785ef0>, 1553885.057639488)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15310377ea10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b35b1c0>, 1553866.710823464)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304b214fd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b65a510>, 1553866.899011728)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304b250b90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b260ad0>, 1553866.997211241)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304b272410>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b261240>, 1553867.395382173)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304b420890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b261710>, 1553867.491274911)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304b2ab0d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bb23150>, 1553868.078148311)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304beee090>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b45c600>, 1553867.330409256)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304b26bf50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b45cd70>, 1553867.420281935)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304b2c7110>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b260050>, 1553867.948997704)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304b2fa050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b2608a0>, 1553868.056176082)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304befd890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304be49160>, 1553868.611007859)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304be635d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304be498d0>, 1553868.417949827)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304be77b10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304be4a040>, 1553868.472666143)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304b272d10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b358670>, 1553868.674211546)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304b432150>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b45dda0>, 1553868.667853715)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153056601090>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b45fc40>, 1553868.731531957)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530566164d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304be4b690>, 1553869.212599715)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153056638a10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304be4baf0>, 1553869.152464627)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304bea2e90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304be4b1c0>, 1553869.083788929)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15305665c7d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304be4bee0>, 1553869.281415124)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153056621c50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530566787c0>, 1553870.362383438)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531075acf50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b65be00>, 1553868.445246343)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304b2ceed0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b559ef0>, 1553868.334943429)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304be5fe90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056679710>, 1553870.066935901)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530566b4790>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056679be0>, 1553869.467826567)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530566c9210>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056679ef0>, 1553869.814485589)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530566a8650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15305667a970>, 1553869.988924666)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15305664ca50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530488b82f0>, 1553870.129454489)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153048888f90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530488b84b0>, 1553869.9372848)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304889bf50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530488b8fa0>, 1553870.420390906)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530488d1210>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530488b9710>, 1553870.316029202)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530488ecb90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530488b99b0>, 1553870.638228026)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15305664ea50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304be4ab30>, 1553870.781944237)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304886d5d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530488b8980>, 1553870.510881324)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530488c5790>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530488bab30>, 1553871.030859421)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153048930950>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bba5630>, 1553871.115383518)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153048911ad0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530488bb9a0>, 1553870.94789125)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153048954890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b55a4a0>, 1553871.440371974)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153048956750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048974c20>, 1553871.630535873)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153048991290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048975320>, 1553871.631809805)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530489b0e50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530488b92b0>, 1553871.773726781)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530489c20d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048976040>, 1553871.884085431)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530489d6150>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048976740>, 1553871.89875139)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530566856d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048976f20>, 1553872.06383715)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153048a13b90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b3586e0>, 1553872.338860997)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153048a32290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530566799b0>, 1553870.893008814)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530489451d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b45fd90>, 1553872.270849752)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530488fadd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15305667ad60>, 1553872.399594923)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304bf01450>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530488ba820>, 1553872.455928697)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304bf62350>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bf1d940>, 1553872.804112312)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153048956390>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bf1e190>, 1553873.069318667)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304bf90b90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bf1e6d0>, 1553872.623128963)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304bf9fa10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bf1ee40>, 1553873.044233928)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304bfbc550>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bf1f5b0>, 1553873.280498789)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304bfbe8d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bf1fd20>, 1553873.582558777)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304bfef610>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bf1fee0>, 1553873.618243967)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530489d6bd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048a48600>, 1553873.734838327)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153048a51450>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048a49400>, 1553874.196606349)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153048a6b3d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048a49630>, 1553873.371646492)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153048a80410>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530566783d0>, 1553874.011479409)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153048aa4f90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048a4a510>, 1553874.057880815)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153048ab3d50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048a4a890>, 1553874.070144345)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153048a36fd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048a4aac0>, 1553874.243990151)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153048ad1ed0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530488b88a0>, 1553873.701101674)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153048af1750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048a4b850>, 1553874.078106664)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304bf397d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048a4ba10>, 1553874.015404069)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153048b09c10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048b24130>, 1553874.579174122)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153048b1c790>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048b24e50>, 1553874.108252967)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153104f6f510>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048b255c0>, 1553875.178516371)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304bfbd510>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048b254e0>, 1553874.895702779)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153048abe1d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530489747c0>, 1553875.48710994)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153104fc49d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304be48ad0>, 1553874.629918808)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153104f6bb90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048b25cc0>, 1553875.243295577)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530489c49d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bf1f310>, 1553875.303620994)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153105000a50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153105034590>, 1553875.431966476)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153105023dd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531050355c0>, 1553875.893951568)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15310504a3d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153105035cc0>, 1553875.763197729)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153104e872d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153105035f60>, 1553875.339709358)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531050496d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153105036740>, 1553875.883719043)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153104eb0e10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15305667b850>, 1553875.813500907)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304bfe1390>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153105037930>, 1553875.83214477)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153104ee3e50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153105035400>, 1553876.254038315)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153104eef390>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048b27f50>, 1553875.640031355)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153105020dd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048b24980>, 1553876.607178128)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304bf92090>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048b26200>, 1553876.827885797)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153104f593d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153104f15cc0>, 1553877.109955526)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153104462d10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048b250f0>, 1553877.372110717)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153104f354d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153104f16660>, 1553876.857891725)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15310447e710>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153104f16890>, 1553877.070112236)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531050554d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531050341a0>, 1553877.235301931)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153104e87dd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153104f152b0>, 1553877.091747048)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531044b6790>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048b27cb0>, 1553877.379143247)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531044edc10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531044e8910>, 1553877.562818719)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15310450d550>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531044e9160>, 1553877.85253453)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531044d7ed0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531044e92b0>, 1553877.275730946)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531045233d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531044e9550>, 1553877.777836195)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153104ec6790>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531044e9710>, 1553877.764366118)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15310453df90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531044e9ef0>, 1553877.834678577)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15310454be10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531044eaf90>, 1553878.465606015)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153104393e90>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531044ebaf0>, 1553878.749243234)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531043c69d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153104f16430>, 1553879.247597349)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531043e8910>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531044e8ad0>, 1553878.858893415)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531044106d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bf1f770>, 1553879.156131396)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15310442ea90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048b26b30>, 1553879.060595966)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531044cd6d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531043d2eb0>, 1553878.991559463)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153103cc16d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531043d08a0>, 1553878.838590456)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531043b9110>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bf1eeb0>, 1553878.81432848)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153104451ed0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153103d042f0>, 1553879.967505234)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153103d0a150>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153103d04ad0>, 1553879.530363729)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153103d306d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153103d05320>, 1553879.833411169)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153103d40f10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153103d05cc0>, 1553880.063409468)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153104423650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153104f14980>, 1553880.681148932)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153103d6ff50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153103d07620>, 1553880.642746359)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153103d63a90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153103d06510>, 1553880.150979204)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153103cc2310>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153103c00130>, 1553880.751425146)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153104532f50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153103c00670>, 1553880.866197197)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153103bf4e10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153103c00440>, 1553881.01534843)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153048aa6fd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531043d02f0>, 1553880.784287783)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153048930910>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531043d34d0>, 1553881.67651572)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153103c7fa50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153103c039a0>, 1553881.360627786)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153103ca0f50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153103d04ec0>, 1553881.236319355)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531038c2f90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531038d4bb0>, 1553882.302545821)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531038efc50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531038d4280>, 1553881.829405296)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153103c8af90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531038d51d0>, 1553882.079900419)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153103910bd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153103c02350>, 1553881.04646677)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153103bcc050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153103c03b60>, 1553881.450392014)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153103cdb150>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153103c03690>, 1553881.540123669)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531038eae90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153103c01fd0>, 1553882.686478706)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153103951590>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531038d67b0>, 1553882.583748034)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153103c2bed0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153103d074d0>, 1553882.464702106)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15310396f3d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531038d7230>, 1553882.539421037)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15310397cad0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531038d74d0>, 1553882.980753767)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15310393ab10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153103d05780>, 1553882.852206734)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531037be250>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531044e8bb0>, 1553882.857383846)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153103cf64d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531038d7a80>, 1553883.034663259)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531038b7210>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531037c1630>, 1553882.796839925)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531037fea50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bf1d710>, 1553883.033469429)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153103821590>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531037c2740>, 1553883.058223585)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153104ea39d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531037c0c20>, 1553883.281425088)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531038cf2d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153103c031c0>, 1553883.025719311)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153103850490>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153103c011d0>, 1553884.180965886)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153103814b90>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531038d4fa0>, 1553884.192037462)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153103880c10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531037c3310>, 1553883.480326151)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15310386a710>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15310389c280>, 1553884.686778059)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153103897410>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153104fead90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bf1f0e0>, 1553875.328464851)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153104feb650>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15310371d850>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531034b9990>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153103d05b70>, 1553886.315798272)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531036cb690>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b45dcc0>, 1553886.564777358)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531034b9d10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531037a3a50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b45d2b0>, 1553886.650572013)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531037a3b50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153103608410>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b45d010>, 1553886.766850845)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153103608210>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531036fb0d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153105000610>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bba71c0>, 1553886.511619536)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531036faf90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153103786e40>, 1553886.70364583)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531050033d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153048a51210>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bba7d90>, 1553886.807728038)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153105038b90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304bf32450>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bba55c0>, 1553886.972580887)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304bf32610>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153048998dd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530566ec5d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bba7700>, 1553886.634103738)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304899ba90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bba4360>, 1553886.595735786)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530566ee690>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153104f1a690>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304be57cd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bba63c0>, 1553887.067922637)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153104f21f50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bba6740>, 1553887.010012181)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304be56d10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304bf60cd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bba4ec0>, 1553886.832978705)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304bf60c10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304b289610>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048a4aa50>, 1553887.971476445)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304b2899d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304b2166d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153103660d90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153103662510>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531035fb450>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153103d3b090>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153103d82010>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15310369ffd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531044aa310>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153103786200>, 1553886.059127876)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531044a8dd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153103d04280>, 1553886.118886902)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153103680590>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153103787690>, 1553887.824543348)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304b216710>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fcd93850>, 1553886.015768335)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153103638dd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531052444b0>, 1553885.981790273)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153103662390>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153103d062e0>, 1553885.94173022)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531035fa2d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153103784a60>, 1553886.2183489)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153103d80590>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b45de80>, 1553886.261711115)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15310369fe10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153103bcdb90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304b36d650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bc28670>, 1553887.604687735)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153103da0610>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bc2bd90>, 1553887.409708295)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304b362910>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304be13150>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304b3f2050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048a4b0e0>, 1553887.473288201)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304be103d0>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bc2ae40>, 1553888.231698205)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153103832090>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304b3154d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bc2ba80>, 1553888.108284001)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304b317710>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304b355c50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bc2bbd0>, 1553887.956815316)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304b355890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304b4a3d50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531044ea350>, 1553888.109460576)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304b4a3d10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304b5c5190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b45f380>, 1553888.289816008)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304b5c4d90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304b484850>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304b5b9a50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531051ad160>, 1553887.992868946)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304345f390>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531051af770>, 1553888.336284741)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304b5bb9d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304b59cb90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531051ae900>, 1553888.230596713)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304b59f090>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304b41ca90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304b432ed0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bba62e0>, 1553888.354210783)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304b41f7d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153103c018d0>, 1553888.573912698)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531038815d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153048a89110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304b520e10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531051af850>, 1553888.753585752)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304be57d90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531051acc90>, 1553888.473981788)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304b520090>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304b6f54d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531051ae7b0>, 1553889.101719526)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304b6f5750>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304b441650>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304b6b2dd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531051af700>, 1553888.452589963)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531034ae250>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531037866d0>, 1553888.948720349)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304b6b3650>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304b63fcd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531050f83d0>, 1553889.574733341)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304b63f090>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304b6a84d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153103787bd0>, 1553889.252560496)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304b6a8150>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153043478450>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bc28f30>, 1553889.359662116)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15310366cc90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304b2b9810>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531050f9b00>, 1553889.493482477)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304b6466d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304b8f2610>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bba4670>, 1553889.965501531)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304b8f0dd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304b679ad0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304b872950>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531050faa50>, 1553889.855175042)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304b678fd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048977150>, 1553889.881805984)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304b871610>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304b8ac4d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304b860d10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531050fb7e0>, 1553889.710242799)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304b8ac390>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531050fb770>, 1553889.824044724)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304b8e0710>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304b5c5d10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304b4b3c50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fc3acd70>, 1553889.651994904)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304b884b10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fc523a80>, 1553889.582898353)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304b4b0d90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304baa5810>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531051afe70>, 1553889.808506706)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304baa5d50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153106c3a690>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304ba84ad0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057f41780>, 1553890.301477591)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153106c3b890>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057f438c0>, 1553889.886067195)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304ba87f50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304b8048d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304ba67e90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057f41e10>, 1553890.480728484)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304b476410>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057f41710>, 1553890.167484143)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304ba66b50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304ba6ed90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057f41400>, 1553890.728361865)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304ba6c050>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fd61d190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fc520de0>, 1553890.757725313)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fd61ff90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153056e918d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fca3e510>, 1553890.651029029)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153056e933d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531064c7410>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fc0f20d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531051aef20>, 1553891.063674981)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531064c4410>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153106b1aad0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fca3d630>, 1553891.175696753)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530579d8ad0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fd5ed050>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531077bb3d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fc7cb150>, 1553890.77802723)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fd5ecfd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fca3eb30>, 1553891.719125134)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531077bbb10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fcc78350>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fc7cbd20>, 1553891.712889579)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fcc78790>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153101bd4650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fc7cacf0>, 1553891.916534532)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fca61390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fc589690>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fe42aa10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531050fb460>, 1553891.682194588)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fc588550>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fc521be0>, 1553891.973192277)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fe42a650>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304337add0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fc2e32d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153105245320>, 1553891.547121254)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304337a250>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531052464a0>, 1553891.64823241)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fc2e0290>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fd363650>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fcb1a050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fca3d010>, 1553891.042510148)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fc0f0d10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bc2b3f0>, 1553891.882840218)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153057bde510>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153105244130>, 1553891.59795374)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fcb1b2d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153057937c10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153105246ac0>, 1553892.140832934)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153057937d10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153056d6ffd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd75ac80>, 1553892.20839956)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153056d6dbd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fd61f050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b262e40>, 1553893.029177723)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fd61f1d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304ba57e10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153056c39290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd759400>, 1553892.857459659)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153056c38e90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153056d25190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153056e42410>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fcee0ec0>, 1553892.491289232)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153056d27d90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fcee08a0>, 1553892.796520909)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153056e434d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153056c703d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fc0f3b10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153102750e50>, 1553893.294400683)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153056d7d890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153056dd6050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153102750de0>, 1553893.193018055)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153056dd4050>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531059ae850>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fcfb5f90>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bc2a4a0>, 1553893.550350865)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531059afb50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd1da0b0>, 1553893.498739661)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304ba87190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fc419490>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057f40c20>, 1553892.543552466)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fd669b50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fca3c2f0>, 1553892.851129398)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304b588610>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fca3c520>, 1553894.048078201)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fc4188d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fd255b50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153105247690>, 1553891.669264943)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fd257210>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fc10e6d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd1db690>, 1553893.26098173)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fd1adf90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fc127010>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fc7d2990>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fc7cbf50>, 1553893.582456815)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fd61cbd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd3473f0>, 1553893.902481075)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fc7d2a10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fd1bf110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15305795ab10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057f40600>, 1553894.201683322)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fd1bc210>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd758520>, 1553893.485451473)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fc41a890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fd1d1090>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bc2a200>, 1553893.898599074)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fd1d3650>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fc256d10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057fdf700>, 1553894.070290771)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153106c78f10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fc98d490>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531023c0390>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057fde660>, 1553894.061321654)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fc98f2d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd13d470>, 1553894.078864828)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531023c0590>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153105f66850>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153057ac1750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd75ac10>, 1553894.300004279)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fd25d5d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd13e120>, 1553894.614054251)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153057ac1b10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153057a362d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057f43070>, 1553894.789818441)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153057a35110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15305797b550>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd13da20>, 1553895.004763285)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15305797b1d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fc378c90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531051d9190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fc72fed0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531050f8600>, 1553893.407034561)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fc0e4790>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd13e660>, 1553895.099923817)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fc378ed0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd1dac10>, 1553893.47520395)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531051d92d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fc07a9d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056cccec0>, 1553895.175130315)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15305795b350>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fc93f450>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153057715910>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303f476270>, 1553895.420943686)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fc93d790>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303f474910>, 1553895.476541781)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304b8ac090>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fc324710>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153057a8b450>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303f4773f0>, 1553895.498442094)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fc325050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303f477460>, 1553895.682660396)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153057a88b90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fc327d50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd6b60b0>, 1553895.507958523)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fc125f50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531021aa490>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bc29e80>, 1553895.330231227)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304b497490>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153101c42210>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303f474bb0>, 1553895.798150978)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fce6ad50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531023c2590>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303f474830>, 1553895.581372879)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531023c2210>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fc551550>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153107716ed0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303f475550>, 1553895.869544961)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fc3fd510>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303f475240>, 1553895.806776057)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531012da6d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153057ab31d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153105a1c190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057fdf230>, 1553895.780580245)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fd1be210>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303f474590>, 1553896.0505474)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153105a1f790>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fd69f310>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fcba1450>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fc0a0d70>, 1553895.929238071)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fd69e790>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530578a4cd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303fcd5e10>, 1553895.941390559)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fcd884d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056ccd390>, 1553894.923818952)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530578a4150>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fce322d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056d69d30>, 1553896.373553621)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fc553910>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153106074050>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fccb5390>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303fcd5d30>, 1553896.500182442)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153106077b50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056d6ba80>, 1553896.723382034)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153106222190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153105951290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056d6b0e0>, 1553896.607939728)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153105950a90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fc755bd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056ccd4e0>, 1553896.909958024)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fc755f50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fc943b90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056cccad0>, 1553896.942986586)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fc942210>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531023c1110>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd2223c0>, 1553897.061792155)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531023c3310>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153057acf8d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd2201a0>, 1553897.000055466)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153057acfd90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153102253950>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153101cb9f10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd220f30>, 1553897.328573011)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fd448090>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303f477a10>, 1553897.461389559)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153101cb8c10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fd181850>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd220980>, 1553897.352624375)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fd181690>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fd71b650>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153057841790>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd222580>, 1553897.249778825)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fcce50d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fc7caba0>, 1553898.447272789)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fc38add0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fd6d2590>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd220d70>, 1553897.942890209)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fd6d1410>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fc9e1a50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fc7ed410>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304ba624a0>, 1553897.852362017)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153056d1ca10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304ba60f30>, 1553897.520032031)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fc7edfd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fcf1da50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304ba61fd0>, 1553897.86329928)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fcf1f390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fcb3c190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303fcd5cc0>, 1553898.163697216)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fcb3c310>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153056e24390>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304ba62970>, 1553898.627623649)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153056e25a50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153056d40c50>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531051cbb50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304ba61860>, 1553898.0249966)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153056d43cd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303f477770>, 1553898.494846389)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531051c8e50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fd021e10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304ba623c0>, 1553898.877723468)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fd020f50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fc741a10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056d69400>, 1553898.740884891)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153107956ad0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fd00fad0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153056d898d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531037c1160>, 1553898.710147403)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fd00d290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd2238c0>, 1553898.966304099)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153056d881d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153100c087d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531037c3d20>, 1553899.170799474)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153100c0ac10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153106ce8b50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd223c40>, 1553899.38881779)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fc144350>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153056de9d10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531066f68d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153106c5dc50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fc9c9390>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304ba61630>, 1553897.549086557)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153105a3b950>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531038d6350>, 1553899.34573959)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15310699ca10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531038d6a50>, 1553898.882905775)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531066f67d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304ba62e40>, 1553897.454666063)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fcb96510>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fc146e50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fc948610>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531038d6740>, 1553899.109390444)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fc9c6bd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531038d6660>, 1553899.514779758)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fc94a1d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fcd42b10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153100b6d6d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b35bcb0>, 1553899.334359245)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fc4e8550>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b358130>, 1553899.511298405)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153100b6d410>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153056d439d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303f306c50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b359b00>, 1553899.443667836)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153056d42190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b359400>, 1553899.864824083)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303f307590>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fca64850>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b359860>, 1553899.711502457)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fca66c50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303f32b9d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303f2f3c50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057fdea50>, 1553899.946597007)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15322cd06490>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b559010>, 1553899.987485236)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303f2f3cd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304b75e890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303f4cce10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b5584b0>, 1553900.121235652)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304b75d150>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056d6ac10>, 1553900.034327792)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303f4ccf10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303f4d7450>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd222040>, 1553900.385865957)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303f4d6f10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304beeb450>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b558590>, 1553900.791986759)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304beea8d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303f4a1790>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303f353ed0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b55a6d0>, 1553900.387031809)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303f4a0350>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531037c0830>, 1553900.559374599)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303f3535d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153101c611d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b659780>, 1553900.467227247)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153056c06c50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153100c0b150>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531037c2580>, 1553900.876407072)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153100c09790>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fc144050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b65a900>, 1553900.739283866)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fc144090>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303f435250>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b65b850>, 1553901.345847413)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303f4341d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153100b6ea50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303f339f50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b6597f0>, 1553901.208818169)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153100b6eb50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b559c50>, 1553901.556035084)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303f339c50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153043050510>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd221710>, 1553900.941511468)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153043050050>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303f5b0750>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303f5a1210>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304be4a900>, 1553901.551365991)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303f5b0590>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b658750>, 1553901.502460972)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303f5a1dd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303f46e150>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b359f60>, 1553901.155934362)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303f5caa90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153103707590>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b35a120>, 1553901.356756259)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303f300bd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303f56c7d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303f526ad0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b857230>, 1553901.582620422)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303f56fc10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b855a90>, 1553901.571551595)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303f527810>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303f451e90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b8554e0>, 1553901.599907697)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303f450910>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303f566350>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303f4066d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b854fa0>, 1553901.96083698)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303f567410>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b55b460>, 1553901.775448147)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303f405090>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153100bddb10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153056ee0c10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b856270>, 1553902.242354165)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153056d8be10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b8551d0>, 1553902.674353243)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153056ee2f50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303fad6490>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bb22d60>, 1553902.045338024)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303fad6810>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303facd110>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b55a660>, 1553902.569366801)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303facc2d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fd331350>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303fa5e4d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b65ba10>, 1553902.380256637)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303fa66450>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bb22820>, 1553902.721879095)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303fa5f350>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303fa86bd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bb20520>, 1553902.50435283)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303f34af90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15310739a1d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bb23e70>, 1553902.470881729)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303f3fadd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153056e98890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303fbcfe50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd221b00>, 1553902.828596336)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303f9fe6d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bb225f0>, 1553902.793700853)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303fa1ee50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303fbc69d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303fb36a90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bd13690>, 1553903.270712561)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303fbc4dd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303fb7c710>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b857460>, 1553903.259222214)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303fb8e6d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303fb7bbd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fc776b50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bd112b0>, 1553903.180030503)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153056e2c950>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fcc0dd10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bd106e0>, 1553903.325960297)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303f5aadd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bd12d60>, 1553903.264491407)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fc7769d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bd134d0>, 1553903.242304782)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fcc0c2d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303fb957d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303fc566d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bd13150>, 1553903.429786267)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303fb842d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bd10ec0>, 1553903.80688431)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303fc55610>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303faa8e10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bd11780>, 1553903.688738356)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303faa9b10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303fb60ad0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bd10980>, 1553903.706708468)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303fb61b50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303fcd18d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bd12200>, 1553904.479488198)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303fcd35d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303fb12190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303fc65ad0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b559320>, 1553904.269680495)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303fb12050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bb233f0>, 1553904.262739573)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303fc66090>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303fc75b90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b659fd0>, 1553904.653232496)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303f44a190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153042d2a9d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bb20ad0>, 1553904.354167682)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153042d28250>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fc825e10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153105263710>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b65a820>, 1553904.646968286)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fc824f50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bca8a60>, 1553904.790744561)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153105260f90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153042eba910>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153042f684d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bb22f20>, 1553904.611819088)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153042ebbcd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153105277d10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bca9710>, 1553904.568368798)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153042f6a810>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bd120b0>, 1553903.36391367)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531052757d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153042d61e10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303f428e90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bcaadd0>, 1553905.024854407)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fc968050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd345ef0>, 1553905.074558315)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303fbb7a50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303faa4190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fd126750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b985860>, 1553905.671214355)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303faa6750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b984d70>, 1553905.753875622)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fd127b90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153042a3f250>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153056e08110>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b985080>, 1553905.144095701)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153042a3ecd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fd58f710>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b986eb0>, 1553905.270630701)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fd58d8d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153056dbad90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b9849f0>, 1553905.533716841)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153056dbb710>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304b181d90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fc3d0550>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b9840c0>, 1553905.199811351)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153100c11c10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b984360>, 1553905.632514145)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153042e82190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153056d5fed0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b9878c0>, 1553905.429996122)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153056d5e350>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153056d10110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fd089850>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b985b70>, 1553906.328515973)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153056d87250>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153042e9e290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bd8ee40>, 1553906.190778782)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153042e9e750>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fcb717d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153057afd590>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b559e10>, 1553906.012680072)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fcb72ed0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bca8440>, 1553905.966074244)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153057afce50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153042a07990>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b35bf50>, 1553905.284668082)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153056e0b0d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b985a20>, 1553906.250819001)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153056d127d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531038d6f90>, 1553906.695211261)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153042a06510>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153056ee3650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bd13000>, 1553906.728747132)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fd5c3cd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153056d51290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bd8d860>, 1553907.131520708)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153042cd05d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531034cd4d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b854c90>, 1553906.723135772)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531034ce750>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15310368a8d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531034dfe50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bd8e350>, 1553906.869378123)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fd125850>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531051511d0>, 1553906.939342898)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531034dfd50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153042a3d990>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153102fc6890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bcaad60>, 1553907.573627416)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153056c7cf10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153105150bb0>, 1553906.990258675)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153102fc6690>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153102fcdc10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153105151be0>, 1553907.26643656)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153102fcd890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153056e131d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bd8cc20>, 1553907.427348493)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153056e11b10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153102fede50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153102ffd690>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153105152890>, 1553907.259589042)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153102fee090>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531051517f0>, 1553907.218317002)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153102ffd490>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153102fd2b10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15310301fd10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153105150830>, 1553907.536285863)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153102fd2e90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153103032a90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153103051ed0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fd57e6d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153105150d70>, 1553907.612766883)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15310301fbd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bd8fb60>, 1553906.093097012)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fd57df90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153105153b60>, 1553907.759522914)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153103032990>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153105152c10>, 1553907.684236411)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153103047190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153103060e50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057e92eb0>, 1553907.925473119)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153103060c50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15310303a250>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153103075750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bd8fd20>, 1553907.961680211)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15310303a490>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b985c50>, 1553908.00571385)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153103075590>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531030827d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057e923c0>, 1553908.296436487)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15310126abd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15310308fcd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153103625910>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bb20fa0>, 1553908.116744533)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15310308ecd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057e93a10>, 1553908.567165638)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153056d5c350>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153103093350>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153100a6eb90>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b7c0670>, 1553908.796282229)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153056e12b10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153100a85ad0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b7c1c50>, 1553908.76103458)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153100a86090>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153102ffdd50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b7c2a50>, 1553908.563186008)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153100a445d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153102fbd050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b7c1b00>, 1553908.908403855)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153042a05890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153100a654d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b7c3620>, 1553908.932298075)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153100a66450>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153100a8ff50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153100ade190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b7c2890>, 1553908.845713562)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153100a8fc90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b7c2e40>, 1553908.997662165)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153100ade890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153100aee450>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bb21a20>, 1553908.853273594)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153100a6e990>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b7c1ef0>, 1553908.917652364)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153100aee310>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153100afb050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b7c1e10>, 1553909.570067661)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153100afaf10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153100b09110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153100b1d790>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd504360>, 1553909.140558226)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153100b09490>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd506040>, 1553909.460890244)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153100b1d590>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153100b2e990>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd507000>, 1553909.443128371)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531034cf390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153100941e50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd507700>, 1553909.634178144)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153100941c50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153100954310>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15310096a290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bd8fc40>, 1553909.447095967)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153100954350>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153105153af0>, 1553909.848397607)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15310096a150>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15310097d250>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd5db4d0>, 1553909.722656261)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15310097d050>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531009571d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303fc54690>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153105153c40>, 1553909.55981662)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153100957410>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531009c6a90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531009d15d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153100bc1400>, 1553910.288601984)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303f4d6fd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fccb39a0>, 1553910.18764293)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531009d1490>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531009f48d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153100bc01a0>, 1553910.737895176)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531009f4e50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153101269350>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153100a10890>\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b7c3850>, 1553910.015134858)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531009a3f90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b7c00c0>, 1553910.225976158)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531009f6610>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fccb38c0>, 1553911.007802197)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153100a10490>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531009dfa10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd42d010>, 1553910.423651144)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531009dec50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153100435a90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd42d780>, 1553910.791763051)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153100435950>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531009f2e50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153100451290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153100bc3930>, 1553910.48564167)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531009f2d10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531079b8590>, 1553910.988628757)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153100451090>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153100478650>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153100b081d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153100b2f750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bd12f90>, 1553911.371428081)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153100b08050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fe39e900>, 1553911.954879459)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153100b2f1d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153100463610>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531004b9010>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fe39d940>, 1553911.044909461)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153100474fd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531004c66d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd5da900>, 1553911.4116206)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531004c5950>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15310096a0d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531004eea10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd42f1c0>, 1553911.331288268)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15310096b450>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fe434ad0>, 1553911.502602587)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531004ee810>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531004f9890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fe437af0>, 1553911.40233584)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531004f9750>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153100516110>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fe435860>, 1553911.558333465)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531004c6690>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153100332010>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd505010>, 1553910.974611099)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153100a78690>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056c534d0>, 1553910.954771554)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531004b94d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153100a26d50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fe39f380>, 1553912.036209035)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153100331e10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd42cb40>, 1553910.376615669)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153100a26c10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531009d2f10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303f5cec80>, 1553911.817311684)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153100a61ed0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531004cef90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153100354910>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303f5cc3d0>, 1553911.825217666)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531004cf350>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303f5ce740>, 1553912.482676751)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531009c7390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531004de290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303f5cd240>, 1553912.257622279)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153100a1ce50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153100371150>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303f5cec10>, 1553912.105066422)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153100371050>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15310036d190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303f5cd390>, 1553912.264469398)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15310036e550>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15310039c510>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303f5cf540>, 1553912.654533556)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15310039d950>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531003877d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531003c82d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303f5cf3f0>, 1553912.901831889)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153100386f50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303f5ce0b0>, 1553912.999573005)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531003c8ad0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153100390050>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531003f0990>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303f5ced60>, 1553912.617836618)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531009dc5d0>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303f529470>, 1553912.6526728)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531003f1c90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531003fda10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303f528ec0>, 1553912.897800776)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531003fd810>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531009b9c50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531004c6ad0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056c52f20>, 1553912.921550173)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153100424cd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303f52b150>, 1553913.20556016)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531004c6cd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153100406cd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531002424d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303f528520>, 1553913.638518949)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153100406e50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303f52b310>, 1553913.070725386)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153100242390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153100261d10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303f52a270>, 1553913.47873368)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531002617d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531009dfb50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153100292710>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303f529b00>, 1553913.312328231)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15310041efd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303f52a820>, 1553913.621808771)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531002925d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531002a3190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531002abed0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303fa9add0>, 1553913.681035036)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531002abdd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153100356dd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531002d42d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153105150ad0>, 1553913.89690438)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153100357650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303fa9a7b0>, 1553913.631019683)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153100514a10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531002e00d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303fa9aeb0>, 1553914.781868471)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531002e0610>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531002f7dd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303fa9bd90>, 1553914.456935168)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531002f7cd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153100324690>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd42d400>, 1553914.143403686)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15310026b350>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531001384d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303fa99940>, 1553914.701525891)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531001387d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153100301b10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303fa9b2a0>, 1553914.30452004)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531003019d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15310030ded0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153100254f90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531003ae510>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fccb2e40>, 1553914.118520801)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531002a1090>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531079bbee0>, 1553913.127036393)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531003add50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303fb47000>, 1553914.367054227)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15310030dfd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303fb452b0>, 1553914.446873646)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531003c9950>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153100387810>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303fb458d0>, 1553914.808844105)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153100386ed0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15310041f150>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd505a90>, 1553915.113872188)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15310041ef10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153100142590>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531001af7d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303fb465f0>, 1553915.068955887)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153100142ed0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303fb45160>, 1553915.279113006)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531001af690>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531001ce2d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303fa9b850>, 1553915.009328761)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531001ce8d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531001de810>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531001fc210>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303fb45240>, 1553915.376558191)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531001fc850>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531001f4290>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531002066d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303fb448a0>, 1553915.829699625)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531001f45d0>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303fb44e50>, 1553915.682537598)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153100207b90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531001c50d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303fbe3e70>, 1553915.709797536)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531001c4f10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153100223850>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303f52bc40>, 1553915.473177611)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153100223950>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530ffe3dd50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530ffe58d50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303fbe3540>, 1553915.868540694)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530ffe3c190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303fbe32a0>, 1553915.835450055)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530ffe58b50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530ffe73e50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd42e6d0>, 1553916.581899227)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530ffe73d50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056c52190>, 1553917.217699381)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530ffe98a10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153100194750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303f5281a0>, 1553915.342219763)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531001ea810>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303f52b930>, 1553914.683026252)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153100502dd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530ffe9a190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303fbe37e0>, 1553916.429148391)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530ffe99f90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530ffea0290>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530ffec0b10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303fbe22e0>, 1553916.258262932)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530ffea0110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530ffec3850>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303f3219b0>, 1553916.496543926)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530ffec3650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303fbe07c0>, 1553916.696721596)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530ffec0710>\n",
      "Evaluating workflow:  38%|███▊      | 1126/3000 [01:47<02:37, 11.87it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  38%|███▊      | 1132/3000 [01:47<02:25, 12.86it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  38%|███▊      | 1135/3000 [01:48<03:27,  9.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  38%|███▊      | 1138/3000 [01:48<03:22,  9.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  38%|███▊      | 1145/3000 [01:49<02:21, 13.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  38%|███▊      | 1150/3000 [01:49<01:47, 17.16it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  38%|███▊      | 1153/3000 [01:49<02:50, 10.86it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  38%|███▊      | 1155/3000 [01:50<02:56, 10.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  39%|███▊      | 1157/3000 [01:50<02:42, 11.35it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  39%|███▊      | 1159/3000 [01:50<03:28,  8.84it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  39%|███▊      | 1161/3000 [01:50<03:06,  9.88it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  39%|███▉      | 1167/3000 [01:51<02:36, 11.72it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  39%|███▉      | 1172/3000 [01:51<01:59, 15.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  39%|███▉      | 1174/3000 [01:51<02:58, 10.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  39%|███▉      | 1176/3000 [01:52<03:10,  9.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  39%|███▉      | 1180/3000 [01:52<03:01, 10.05it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  39%|███▉      | 1182/3000 [01:52<03:07,  9.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  39%|███▉      | 1184/3000 [01:52<02:50, 10.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  40%|███▉      | 1188/3000 [01:53<02:38, 11.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  40%|███▉      | 1192/3000 [01:53<02:30, 11.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  40%|███▉      | 1196/3000 [01:54<03:20,  9.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  40%|████      | 1200/3000 [01:54<03:11,  9.40it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  40%|████      | 1208/3000 [01:54<02:06, 14.16it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  40%|████      | 1210/3000 [01:55<02:13, 13.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  40%|████      | 1212/3000 [01:55<02:30, 11.89it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  40%|████      | 1214/3000 [01:55<02:51, 10.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  41%|████      | 1217/3000 [01:56<03:26,  8.63it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  41%|████      | 1219/3000 [01:56<02:58,  9.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  41%|████      | 1223/3000 [01:56<02:34, 11.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  41%|████      | 1225/3000 [01:56<02:41, 10.96it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  41%|████      | 1230/3000 [01:57<02:02, 14.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  41%|████      | 1232/3000 [01:57<03:14,  9.09it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  41%|████      | 1236/3000 [01:57<02:52, 10.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  41%|████▏     | 1240/3000 [01:58<02:30, 11.67it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  41%|████▏     | 1242/3000 [01:58<02:56,  9.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  41%|████▏     | 1244/3000 [01:58<02:34, 11.36it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  42%|████▏     | 1249/3000 [01:58<02:29, 11.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  42%|████▏     | 1251/3000 [01:59<02:40, 10.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  42%|████▏     | 1255/3000 [01:59<02:18, 12.58it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  42%|████▏     | 1257/3000 [01:59<02:27, 11.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  42%|████▏     | 1262/3000 [02:00<02:31, 11.49it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  42%|████▏     | 1264/3000 [02:00<02:36, 11.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  42%|████▏     | 1266/3000 [02:00<02:48, 10.27it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  42%|████▏     | 1270/3000 [02:00<02:47, 10.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  42%|████▏     | 1274/3000 [02:01<03:13,  8.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  43%|████▎     | 1277/3000 [02:01<02:36, 11.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  43%|████▎     | 1282/3000 [02:01<02:09, 13.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  43%|████▎     | 1284/3000 [02:02<02:14, 12.72it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  43%|████▎     | 1286/3000 [02:02<03:03,  9.36it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  43%|████▎     | 1289/3000 [02:02<02:47, 10.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  43%|████▎     | 1294/3000 [02:03<02:33, 11.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  43%|████▎     | 1298/3000 [02:03<02:48, 10.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  43%|████▎     | 1301/3000 [02:03<02:12, 12.84it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  43%|████▎     | 1303/3000 [02:04<03:00,  9.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  44%|████▎     | 1309/3000 [02:04<02:16, 12.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  44%|████▍     | 1313/3000 [02:04<02:10, 12.96it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  44%|████▍     | 1315/3000 [02:05<02:48,  9.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  44%|████▍     | 1317/3000 [02:05<02:27, 11.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  44%|████▍     | 1321/3000 [02:05<02:34, 10.83it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  44%|████▍     | 1323/3000 [02:05<02:54,  9.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  44%|████▍     | 1327/3000 [02:06<02:38, 10.55it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  44%|████▍     | 1331/3000 [02:06<02:45, 10.09it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  44%|████▍     | 1335/3000 [02:07<02:59,  9.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  45%|████▍     | 1338/3000 [02:07<02:39, 10.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  45%|████▍     | 1343/3000 [02:07<02:16, 12.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  45%|████▍     | 1347/3000 [02:08<02:27, 11.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  45%|████▍     | 1349/3000 [02:08<02:41, 10.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  45%|████▌     | 1352/3000 [02:08<02:13, 12.38it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  45%|████▌     | 1356/3000 [02:09<03:01,  9.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  45%|████▌     | 1358/3000 [02:09<02:40, 10.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  45%|████▌     | 1360/3000 [02:09<03:41,  7.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  45%|████▌     | 1361/3000 [02:09<03:42,  7.38it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  46%|████▌     | 1368/3000 [02:10<02:17, 11.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  46%|████▌     | 1371/3000 [02:10<02:14, 12.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  46%|████▌     | 1373/3000 [02:10<03:03,  8.87it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  46%|████▌     | 1375/3000 [02:11<03:32,  7.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  46%|████▌     | 1378/3000 [02:11<03:32,  7.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  46%|████▌     | 1380/3000 [02:11<02:55,  9.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  46%|████▌     | 1387/3000 [02:12<01:45, 15.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  46%|████▋     | 1389/3000 [02:12<02:00, 13.36it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  46%|████▋     | 1391/3000 [02:12<02:23, 11.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  46%|████▋     | 1393/3000 [02:12<03:18,  8.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  47%|████▋     | 1396/3000 [02:13<03:41,  7.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  47%|████▋     | 1400/3000 [02:13<02:44,  9.74it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  47%|████▋     | 1409/3000 [02:13<01:12, 21.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  47%|████▋     | 1412/3000 [02:14<03:13,  8.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  47%|████▋     | 1415/3000 [02:15<03:24,  7.74it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  47%|████▋     | 1420/3000 [02:15<02:57,  8.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  48%|████▊     | 1427/3000 [02:16<01:45, 14.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  48%|████▊     | 1430/3000 [02:16<01:40, 15.55it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  48%|████▊     | 1433/3000 [02:16<02:23, 10.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  48%|████▊     | 1437/3000 [02:17<03:11,  8.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  48%|████▊     | 1441/3000 [02:17<02:13, 11.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  48%|████▊     | 1445/3000 [02:17<02:04, 12.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  48%|████▊     | 1447/3000 [02:18<01:53, 13.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  48%|████▊     | 1451/3000 [02:18<02:13, 11.61it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  48%|████▊     | 1453/3000 [02:18<02:13, 11.57it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  48%|████▊     | 1455/3000 [02:19<04:00,  6.42it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  49%|████▊     | 1457/3000 [02:19<03:37,  7.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  49%|████▉     | 1464/3000 [02:19<02:11, 11.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  49%|████▉     | 1469/3000 [02:20<01:51, 13.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  49%|████▉     | 1474/3000 [02:20<01:37, 15.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  49%|████▉     | 1476/3000 [02:21<03:59,  6.35it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  49%|████▉     | 1480/3000 [02:21<02:40,  9.47it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  50%|████▉     | 1485/3000 [02:21<02:17, 10.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  50%|████▉     | 1488/3000 [02:22<02:10, 11.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  50%|████▉     | 1491/3000 [02:22<02:05, 12.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  50%|████▉     | 1493/3000 [02:22<02:56,  8.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  50%|████▉     | 1497/3000 [02:23<02:48,  8.92it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  50%|████▉     | 1499/3000 [02:23<02:25, 10.29it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  50%|█████     | 1504/3000 [02:23<02:12, 11.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  50%|█████     | 1506/3000 [02:24<02:22, 10.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  50%|█████     | 1511/3000 [02:24<01:57, 12.72it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  50%|█████     | 1515/3000 [02:24<02:06, 11.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  51%|█████     | 1520/3000 [02:25<02:26, 10.09it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  51%|█████     | 1522/3000 [02:25<02:14, 11.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  51%|█████     | 1524/3000 [02:25<02:14, 11.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  51%|█████     | 1526/3000 [02:25<02:33,  9.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  51%|█████     | 1529/3000 [02:26<02:55,  8.40it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  51%|█████     | 1533/3000 [02:26<01:47, 13.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  51%|█████     | 1535/3000 [02:26<02:28,  9.87it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  51%|█████▏    | 1539/3000 [02:27<02:30,  9.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  51%|█████▏    | 1541/3000 [02:27<02:11, 11.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  52%|█████▏    | 1545/3000 [02:27<02:17, 10.57it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  52%|█████▏    | 1548/3000 [02:27<01:55, 12.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  52%|█████▏    | 1550/3000 [02:28<01:58, 12.27it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  52%|█████▏    | 1554/3000 [02:28<02:13, 10.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  52%|█████▏    | 1556/3000 [02:28<02:44,  8.80it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  52%|█████▏    | 1562/3000 [02:29<02:00, 11.97it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  52%|█████▏    | 1567/3000 [02:29<02:18, 10.35it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  52%|█████▏    | 1572/3000 [02:30<01:56, 12.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  52%|█████▏    | 1574/3000 [02:30<02:13, 10.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  53%|█████▎    | 1578/3000 [02:30<01:56, 12.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  53%|█████▎    | 1580/3000 [02:31<02:04, 11.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  53%|█████▎    | 1582/3000 [02:31<02:36,  9.06it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  53%|█████▎    | 1584/3000 [02:31<02:53,  8.16it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  53%|█████▎    | 1586/3000 [02:31<02:45,  8.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  53%|█████▎    | 1589/3000 [02:32<02:20, 10.07it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  53%|█████▎    | 1591/3000 [02:32<02:57,  7.92it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  53%|█████▎    | 1597/3000 [02:32<01:59, 11.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  53%|█████▎    | 1599/3000 [02:33<02:02, 11.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  53%|█████▎    | 1603/3000 [02:33<02:53,  8.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  54%|█████▎    | 1606/3000 [02:34<02:41,  8.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  54%|█████▎    | 1609/3000 [02:34<02:55,  7.93it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  54%|█████▎    | 1611/3000 [02:34<02:55,  7.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  54%|█████▍    | 1616/3000 [02:35<01:52, 12.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  54%|█████▍    | 1621/3000 [02:35<01:25, 16.11it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  54%|█████▍    | 1623/3000 [02:35<01:44, 13.14it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  54%|█████▍    | 1625/3000 [02:36<02:37,  8.74it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  54%|█████▍    | 1627/3000 [02:36<02:53,  7.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  54%|█████▍    | 1630/3000 [02:36<02:26,  9.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  55%|█████▍    | 1635/3000 [02:36<01:51, 12.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  55%|█████▍    | 1639/3000 [02:37<02:07, 10.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  55%|█████▍    | 1643/3000 [02:37<01:29, 15.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  55%|█████▍    | 1645/3000 [02:37<02:26,  9.27it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  55%|█████▍    | 1648/3000 [02:38<01:56, 11.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  55%|█████▌    | 1653/3000 [02:38<02:12, 10.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  55%|█████▌    | 1655/3000 [02:38<02:14, 10.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  55%|█████▌    | 1657/3000 [02:39<02:57,  7.58it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  55%|█████▌    | 1664/3000 [02:39<01:50, 12.09it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  56%|█████▌    | 1666/3000 [02:40<01:59, 11.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  56%|█████▌    | 1668/3000 [02:40<01:52, 11.88it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530ffde0810>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056ee5940>, 1553918.164780739)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530ffde0910>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530ffd9fd50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056e05400>, 1553919.669436141)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530ffce6090>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530ffdadd10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056e06f90>, 1553919.349379423)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530ffb35b50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530ffe04950>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15310389f230>, 1553919.775808691)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530ffd01410>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530ffcdf9d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056e04910>, 1553918.731898108)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530ffcdf890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530ffc1f150>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303fbe2c10>, 1553920.480356806)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530ffc1f050>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530ffa5edd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530ffbde350>, 1553920.464081066)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530ffa5ea10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530ffa76bd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530ffadb770>, 1553922.385420751)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530ffaf9fd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530ff9feed0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530ffbdc130>, 1553923.185449874)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530ff9ff110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530ff547850>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530ff6d2740>, 1553923.967687052)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530ff6dca50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530ff539210>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530ff6d0590>, 1553924.139988592)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530ff539cd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530ff6bb290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530ff5c8520>, 1553926.141851619)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530ff709e10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530ff6c7d90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530ff6d0830>, 1553923.327804747)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530ff6c7c50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530ffa1aad0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530ff5c9f60>, 1553925.166091057)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530ffa19c10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fea6af90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530feabc210>, 1553926.009846549)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fea6ad50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fea9ba90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530feabd630>, 1553926.191903511)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530ff60ae90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fe9306d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530feabe9e0>, 1553925.877068569)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fe930550>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530feb1a8d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530feabe2e0>, 1553926.574866855)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530feb135d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fe94bd50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fe9b2120>, 1553927.643728015)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fdc7cd50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530feaeb5d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fe9b3540>, 1553927.650455008)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530feaeb650>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530ff93c750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fe9b2b30>, 1553928.310792835)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fdce09d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530ff6a1190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530feabec10>, 1553926.247859154)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530ff59d890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530feb13fd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fdcff380>, 1553928.334016358)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530ff5def50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fdcb0c10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530ff9dc130>, 1553927.460056476)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fdcb3fd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fdafea50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fdae8de0>, 1553929.743984371)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fdafea10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fdb1e5d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530ff5c8590>, 1553930.151287826)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fdb1e810>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fdc8c7d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530ff5c9780>, 1553930.664523568)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fd97c5d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fdb666d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fdaeb070>, 1553930.469200366)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fdb66810>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fdae1250>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd9d7070>, 1553931.557143403)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fd888210>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fda39910>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fdbf9940>, 1553931.549172642)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fda3a390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fd95e610>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd8d66d0>, 1553932.332959119)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fd95e510>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fd807650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd7d1240>, 1553934.419605789)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fd807510>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fdae0410>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd7d1c50>, 1553933.461348673)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fda47390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fda48490>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd7d2510>, 1553933.365013801)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fda48990>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fd95c490>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd8d64a0>, 1553932.283609805)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fd95cb50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153057428310>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd7d3a10>, 1553934.376554014)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153057423a50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fe94a690>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fdcfe0b0>, 1553934.361288278)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15305742a0d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530574231d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153057468310>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd7d3070>, 1553934.355733453)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fd91dfd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15305744cb40>, 1553934.370671916)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530574639d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530574793d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15305744cd70>, 1553934.397826714)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153057479a50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fdbf6b90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530574a4f90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15305744d010>, 1553934.395041693)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fdbf56d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530574b4610>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd7d3e00>, 1553934.7232965)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530574b4410>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530574e03d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15305744e7b0>, 1553934.949903574)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153057439250>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530574fac50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153057329310>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15305744fa80>, 1553934.92536944)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530573291d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530574f95d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15305744fc40>, 1553934.821446651)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fdae17d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fd863390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153057349350>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15305734c360>, 1553935.587715926)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fd863210>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15305734c520>, 1553935.188263674)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153057349210>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530573584d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd7d3cb0>, 1553934.500444566)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530574a4d90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fdbfbd20>, 1553934.886464321)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530574fb1d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15305734cd00>, 1553935.425598429)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15305735af10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15305735cf50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153057378ed0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531001ceb50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530ffefed10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530ffedf290>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fff28490>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fff2a950>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530ffd566d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530ffd74450>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530ffd8c390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530ffeeb810>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530ffd4cc50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530ffd3f090>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530ffda93d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530ffdad2d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530ffdb73d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530ffe99fd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530ffdfbb10>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530ffe1c890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530ffc58c90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530ffc6f090>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530ffc82010>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530ffdf8810>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530ffc9d0d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530ffc6df50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530ffcb38d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530ffc51a10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530ffcc9d90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530ffd00710>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530ffd3f310>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530ffd27ad0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530ffb4fd10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530ffc82810>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530ffbf8950>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530ffb75950>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530ffb87a90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530ffbd0550>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530ffb9fa50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530ffc04a10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530ffba6dd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530ffa40c50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530ffa76ed0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530ffa8c2d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530ffaaf110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fff0cd90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530ffae9150>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530ffb94310>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530ffb08e90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530ffad2610>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530ffb23ad0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530ff93de10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530ffa45990>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530ff992c50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530ff991ad0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530ff9b5ed0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530ff9e66d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530ff9fce90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530ffa2b150>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530ff637cd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530ff65d890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530ffb0a210>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530ff680c50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530ff68ff90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530ff987e50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530ff9d6550>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530ffb15710>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530ff6fa5d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530ff7254d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530ff5454d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530ff559410>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530ff578f50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530ff59d850>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530ff6ed650>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530ff5df0d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530ff707ed0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530ff60a3d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fea5e550>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fea90a90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530ff574ad0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530feae0d90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530feae9f10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530feb10410>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fe92a450>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530ff5de9d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530ff613f50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530ff5ee4d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fe9436d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fe9654d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fe9a54d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fe9d2090>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fe9e46d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fe9c4290>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fdc76450>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530feb01c10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fdcda4d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fdd08cd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fe9d3ad0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fdd18b10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fdd384d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fdd44710>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fdd63490>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fdb7e6d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fdba0190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fdbbb010>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fdb7b650>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fdbf5750>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fdc8c710>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fdba0790>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fdc29490>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fdc454d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fe92b190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fda76490>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fdc5ce10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fdab1d50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fdae02d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fda88090>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fdb3f310>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fe9a55d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fdb5b410>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fd9877d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fd9a6a10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fdc6f710>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fd9cd990>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fdbd3d10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fda2e390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fda47610>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fd9ee2d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fd87a390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fd898850>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fd871850>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fdd4f410>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fd9f2590>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fd871f90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fd8d9890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fd91e190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fd935750>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fd8f0b10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fd94f690>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fd7ac6d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fda6d190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fd7ad8d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fd7eb2d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fd8315d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fd7d6d10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fd851a50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fd96e710>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153057444150>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530ffbdea50>, 1553934.052325935)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fda47690>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15305734cec0>, 1553935.594860176)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15305735d290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15305734d0f0>, 1553935.557806134)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153057379610>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303fa9b310>, 1553917.047180922)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530ffeb7690>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303f3235b0>, 1553916.91951974)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530ffeff3d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fccb2430>, 1553916.689711912)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fff0bf50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303f322f20>, 1553916.913566918)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fff28290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303f322b30>, 1553917.113455371)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fff2a810>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303f321400>, 1553917.190881489)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530ffd56250>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303f5cc210>, 1553917.558597244)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fff0c550>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303fa9ae40>, 1553917.446062126)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530ffd8bf90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303f323af0>, 1553917.034195539)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530ffeeb6d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303fb45fd0>, 1553917.078756582)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530ffd4cf90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056ee6820>, 1553917.571035358)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531001c4c50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056ee54e0>, 1553917.550071089)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530ffda91d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056ee60b0>, 1553917.521960139)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530ffdadb50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056ee78c0>, 1553917.742724202)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530ffdb7290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056ee6900>, 1553918.113744461)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15310034f190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303f3208a0>, 1553918.165982197)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530ffdfb9d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303f321be0>, 1553919.141637384)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fff0cdd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303fce7850>, 1553918.602394325)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530ffc59210>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303fce6970>, 1553918.928486699)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153100182110>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303fce7000>, 1553918.73990664)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530ffc81ed0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303fce67b0>, 1553919.048691028)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530ffe24490>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303fce74d0>, 1553918.802724487)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530ffc9cf90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303f321630>, 1553919.501572828)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530ffc6e110>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056e062e0>, 1553918.791717196)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530ffcb3dd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056ee6c10>, 1553918.630649471)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530ffc52110>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056e07e70>, 1553918.883579236)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530ffcc9b90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303fb46f20>, 1553919.513661607)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530ffd00e50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056e07620>, 1553919.239069628)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530ffd3f250>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056e07d20>, 1553919.755038023)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530ffd27990>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056e074d0>, 1553919.812345745)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530ffb4fbd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056e042f0>, 1553920.533194331)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530ffc82910>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303f321fd0>, 1553920.589872056)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530ffbf9010>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056ee5c50>, 1553919.848035127)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530ffb757d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15310389df60>, 1553919.979655825)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530ffb87950>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15310389c0c0>, 1553920.498367817)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530ffbd0c10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530ffbdc600>, 1553921.011509181)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530ffb9dd50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530ffbdcd70>, 1553920.496784448)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530ffc04810>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530ffbdcf30>, 1553920.692506385)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530ffbbd890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530ffbdcbb0>, 1553920.767308711)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530ffb68f50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530ffbdd8d0>, 1553920.961511444)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530ffa770d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530ffbdf070>, 1553921.239792404)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530ffbcca10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530ffbdfee0>, 1553921.675794699)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530ffaaecd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056ee4d70>, 1553921.32615397)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530ffa51e90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303fce7c40>, 1553921.350753773)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530ffae9010>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530ffad8d00>, 1553921.872140922)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530ffb951d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056ee4440>, 1553921.603146773)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530ffb08c90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303f321390>, 1553921.494020666)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530ffad54d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056e06ac0>, 1553922.306753341)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530ffb23ed0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530ffad9b00>, 1553921.73704456)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530ff93e510>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303fbe24a0>, 1553920.710264873)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530ffa45cd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530ffadb2a0>, 1553922.275856891)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530ff992f90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530ffadb930>, 1553922.677853578)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530ff99c1d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530ffadbc40>, 1553922.578666736)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530ff9b5d90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530ff9dcc20>, 1553922.235997993)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530ff9e7a50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303fce7070>, 1553922.007098137)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530ffb85410>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b7c1d30>, 1553922.861109826)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530ffa2af50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530ff9ddda0>, 1553922.441317446)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530ff637b90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530ff9de510>, 1553923.602475741)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153100451390>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530ff9dec80>, 1553922.968888684)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530ffb0a4d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530ff9deeb0>, 1553923.11127176)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530ff680a90>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530ff9df7e0>, 1553923.401955811)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530ff68fe90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530ffada820>, 1553922.188616866)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530ff987d10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056e049f0>, 1553922.501814865)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530ff9d6c10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530ff9dfbd0>, 1553923.696632313)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530ffb14e90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530ff9df9a0>, 1553923.603778263)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530ff6fa450>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530ff9ddbe0>, 1553923.90209392)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530ff724750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530ff6d23c0>, 1553923.955256924)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530ff545650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303fce7ee0>, 1553923.966116034)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530ff5592d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530ff6d3380>, 1553924.49710405)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530ff579490>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303f323150>, 1553924.526863036)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530ff59d490>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530ff6d3ee0>, 1553924.205747758)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530ff6eda50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530ff6d15c0>, 1553924.519892018)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530ff5def10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530ffad9470>, 1553923.956399415)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530ff707d90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530ff5c84b0>, 1553924.803792235)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530ff60a8d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530ff5cacf0>, 1553925.459957123)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fea5e350>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530ff5cb460>, 1553925.347735493)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530ffcf8b50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530feabcad0>, 1553925.979488548)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530ff574890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530feabd2b0>, 1553925.91724791)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fea684d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530feabd470>, 1553925.827888762)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530feae9590>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530feabd7f0>, 1553925.804506012)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530ff6dd590>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530feabe0b0>, 1553925.901150352)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fe92a250>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530ff5ca120>, 1553924.963439301)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530ff5dead0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530ff5ca890>, 1553924.844190931)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530ff613990>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530ff5cb620>, 1553925.398045569)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530ff5ee450>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530feabf150>, 1553926.431081476)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fe9420d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530feabf310>, 1553926.592178742)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530ff579c10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fe9b01a0>, 1553926.617014582)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fe9a6650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530ff5c82f0>, 1553926.516339072)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fe9d0490>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fe9b1400>, 1553926.907454796)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fe9e5890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fe9b15c0>, 1553926.853441616)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fe9c4790>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fe9b1da0>, 1553927.915796499)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fdc76310>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fe9b2200>, 1553927.930930282)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530feb01050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530ff5cbe00>, 1553927.51726981)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fdcda290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530ff6d2900>, 1553927.688526833)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530feaa9250>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530ff5c9fd0>, 1553927.329614646)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fea0e6d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530ff6d1940>, 1553928.018681446)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fdd189d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fdcfd390>, 1553927.992833413)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fdd39fd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530feabee40>, 1553927.98169878)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fdd45410>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530feabe970>, 1553928.470995497)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fdd639d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530ff6d0f30>, 1553928.590445215)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fdb7f990>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fdcfedd0>, 1553928.043665637)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fdba0f10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fdcff540>, 1553928.196514732)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fdbbb510>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fdcffb60>, 1553928.623928524)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fdb7b190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fdbf83d0>, 1553928.733009448)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fdbf5550>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fdbf99b0>, 1553929.084063271)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fdc8cc90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fe9b0fa0>, 1553928.773530851)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fdba0490>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fdbf9390>, 1553929.447310257)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fdc292d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fdcfc360>, 1553929.627472095)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fdc46450>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fe9b38c0>, 1553929.32727248)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fea5add0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fdbfac80>, 1553929.500668238)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fda76c10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fdbfb770>, 1553929.761992813)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fdc5cd90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fdbfb9a0>, 1553929.735448036)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fdab1c10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fdae8280>, 1553929.793940065)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fdae07d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fdae87c0>, 1553929.685485297)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fda88690>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fdae84b0>, 1553930.057569506)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fdb3f1d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fdae9ef0>, 1553930.42968466)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fdafd890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fdae90f0>, 1553930.390859731)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fdb5b650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fdaeae40>, 1553930.385413535)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fd987350>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fdaeb620>, 1553930.639274328)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fd9a68d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fdae8050>, 1553930.602016888)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fdc6fd90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fdaebf50>, 1553931.032498576)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fd9cd790>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fdbfa200>, 1553931.318570053)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fdc2a7d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fdaeb930>, 1553931.215152232)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fda2ead0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fe9b1fd0>, 1553932.140571211)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fda47210>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd9d46e0>, 1553931.466184161)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fd9ee350>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd9d6e40>, 1553931.730035501)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fd8784d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fdbfb380>, 1553931.726051648)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fd8986d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd9d5010>, 1553932.302313225)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fd8c0ed0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd8d5320>, 1553932.289562506)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fd987390>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fdaea510>, 1553931.38688231)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fd9f1210>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd9d6ba0>, 1553931.21917657)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fd872490>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fdcfdd30>, 1553931.966088623)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fd8d9610>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd8d54e0>, 1553932.35909633)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fd91df90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd8d5b00>, 1553932.080577123)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fd935550>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd8d43d0>, 1553932.491370735)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fd898d10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd8d4c90>, 1553932.683375673)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fd94f150>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd8d7930>, 1553933.178250381)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fd9cc710>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd8d7d20>, 1553933.141720752)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fda6c610>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd8d7e70>, 1553933.183053021)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fdbbbe90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd7d0bb0>, 1553933.379656723)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fdcbce10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd8d73f0>, 1553933.48182822)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fd831410>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd9d4bb0>, 1553933.087948998)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fd7d6fd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd7d2350>, 1553933.451803071)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530ffb4d4d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15305744c600>, 1553934.596660205)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fdc1c890>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530ffdb4d10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fea24d10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fda73710>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530ff9dfee0>, 1553923.186628896)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530ffdb7390>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530ff5c8d00>, 1553925.155122628)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fea24b10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fdcfc440>, 1553929.754078864)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fda73510>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fdb6ac90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530573e2f10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15305732bb90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530ff5cb7e0>, 1553936.230968198)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530573e23d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530ff6d03d0>, 1553936.185329798)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15305732b1d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15305723b710>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15305742a590>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530ff9decf0>, 1553935.868347033)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153057257dd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530ffadb0e0>, 1553936.290942689)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fd84e950>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fdbc0a50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056e07070>, 1553936.104919332)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fe955890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530ff582110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fead32d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15305744da20>, 1553936.795562064)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530ff577ad0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303f3224a0>, 1553936.259326328)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530ff53bad0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fea7ba50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530ff721410>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303f322740>, 1553936.31888393)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530573f37d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd9d4910>, 1553936.377906036)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530ff721c50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153057349390>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056c502f0>, 1553936.572029631)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530feb10d10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530ffa959d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056c539a0>, 1553937.185949893)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530ffc1dfd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530ffdaeb10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530ffddadd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd9d5be0>, 1553936.962027001)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530ffcbe5d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd9d5a90>, 1553937.008602025)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530ffddac50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530ffc49dd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530ffd4c6d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531079bbf50>, 1553938.111303739)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15305736d4d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd8d79a0>, 1553937.305132217)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530ffd4cbd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15305723a3d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530ffe899d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530feabf1c0>, 1553936.289129212)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fdb695d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530573f3bd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fdc1ed90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530573d69d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153057244110>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530ff6d11d0>, 1553935.93770031)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530573d7410>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530ff5cb2a0>, 1553936.122583097)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fdc1f310>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd7d2580>, 1553937.578631166)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530573c74d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057e92e40>, 1553937.211093127)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530ffe8ab90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bd8ec80>, 1553936.156828284)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530573f3a90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530ff6d1a90>, 1553935.903090083)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530572444d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530ffe64a10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057e92eb0>, 1553937.694247523)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530ffe67e10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530ffed5510>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fdcfe270>, 1553937.850243127)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530ffed41d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530ffea3d10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531001fe250>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd7d38c0>, 1553938.207360779)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530ffea3690>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bd8d9b0>, 1553938.325496642)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531001fdd90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530ffeb4350>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531001967d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bd8c0c0>, 1553937.820547612)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fd89a3d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bd8f850>, 1553938.257474118)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531001941d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153100180090>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153100158490>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd8d5ef0>, 1553937.824891634)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153100219c50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057e914e0>, 1553938.282923083)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153042873dd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304287a350>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531003230d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bd8e580>, 1553937.829013563)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fdc74650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bd8d550>, 1553937.872609399)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153100131cd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531002bea90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bd8f070>, 1553938.462102465)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531002bfb90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531002ab6d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bd8f690>, 1553938.320333074)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531002d7f10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531002cbc10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15305744f000>, 1553938.309767082)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531002cb210>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531002e03d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153100254410>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bd8f460>, 1553938.815948072)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530ffe9ac10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd2232a0>, 1553939.164059851)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153100256150>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15310027fb90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd221a20>, 1553938.913062688)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153100405550>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd220910>, 1553939.217986682)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15310027f4d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15310040fe90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd2219b0>, 1553939.038799125)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15310040f4d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531003ae9d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd220d00>, 1553939.414643024)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531003d8cd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153100387910>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531003d8f90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15310036f310>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bd8d010>, 1553939.406604794)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153100387090>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056ccc980>, 1553939.626279039)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531003bacd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531003ca310>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056ccde80>, 1553940.126941472)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531003ca850>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531004faf90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153100347410>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd220590>, 1553939.5546965)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531004f8b90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056dd9860>, 1553939.975974486)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153100515690>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15310049ee50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd1da430>, 1553939.781957184)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15310049f290>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531004dc250>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153100453090>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bd8fe70>, 1553939.92172769)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153100460b10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bd8e0b0>, 1553940.565956753)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153100453210>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530ffc80550>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153100448490>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056c51a90>, 1553940.479653772)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fdd4da90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd6b62e0>, 1553940.257967093)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153100449850>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153100436710>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531002a0610>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd6b4c90>, 1553940.365555882)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153100477cd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd6b6ba0>, 1553940.154753037)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531004b8850>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531009d0310>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056ccd010>, 1553939.882350371)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15310036e650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd6b4a60>, 1553940.38363984)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531003fd390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15310097c1d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056e79be0>, 1553940.806712911)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15310097c2d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531009825d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057f42ac0>, 1553941.172635049)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153100982450>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531009a6850>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd220c20>, 1553941.02451863)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15310027e0d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530ffe8abd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15310093e350>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056c51470>, 1553941.635903351)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531004c6a50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531027510f0>, 1553941.143349429)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15310093dd50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153100ad1a10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fc520de0>, 1553941.534311678)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153100ad1710>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153100a86210>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153100b08610>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fc5239a0>, 1553941.177999078)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153100a849d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531051af000>, 1553941.65409203)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153100ad3590>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153100a6e510>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056ccc3d0>, 1553941.679298423)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153100a6c4d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153100a94ed0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bd8f930>, 1553941.436625979)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153100a974d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153042883050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531051af310>, 1553941.692512526)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530ffbbcf10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153103080590>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153103030850>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531051aec80>, 1553941.881803077)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531030805d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531051af4d0>, 1553941.995894551)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153103030910>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153103038710>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531051acf30>, 1553941.598540646)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531030504d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531030769d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531051ae7b0>, 1553941.696442694)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531004366d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153102fc5d50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd346f90>, 1553942.404863485)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153102fc5c50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fca59dd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153103012490>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd344360>, 1553942.02237397)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153102fbe7d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531052470e0>, 1553942.003881104)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153103011690>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531004c6090>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153043019750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd7d0910>, 1553942.302911275)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153103076c10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531051adda0>, 1553942.177793638)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531009a3d90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fc630750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057f42ba0>, 1553942.662120692)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fc632a10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153056cc66d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153056cb5490>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fc522510>, 1553942.544478629)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153056cb54d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fc09e8d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303fcddf10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15310389eeb0>, 1553942.919212569)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153057affdd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15310389fa10>, 1553943.110891607)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153100abaed0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153042d601d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153103074450>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bd8f540>, 1553943.446152066)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153103074050>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153042d60c50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153042fa2bd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057f40d00>, 1553943.404065942)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303fb8f590>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304ba60bb0>, 1553943.50490288)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153042fa3650>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303fbd5ed0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153105247cb0>, 1553942.979381472)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153056cc6b50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531051af2a0>, 1553943.241975169)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153042d62a90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd6b50f0>, 1553943.557747141)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303fbd4450>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303fa1d150>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153100adfc50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304ba63000>, 1553943.168863876)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303fa1ee50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304ba62510>, 1553943.960278986)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153100a65110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fc632bd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304ba632a0>, 1553943.631726988)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530574071d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303fb666d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fc69e890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304ba627b0>, 1553944.050578983)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153042f6a390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303fbdc890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304ba62cf0>, 1553944.487688804)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303fbdc4d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303fb97a10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303fb9cc50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b2615c0>, 1553943.782299671)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fe3ef1d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153105247700>, 1553943.760939457)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303fb9f9d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303fa0ec10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b262120>, 1553944.016093286)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303fa0c490>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304301bad0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b2624a0>, 1553945.272308727)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304301b4d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153042a04850>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303fc3aed0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b261940>, 1553944.8391932)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153100254b90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b263f50>, 1553944.4164881)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303f586bd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303f53b410>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b263930>, 1553944.82969282)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303f53a250>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303f594fd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b261080>, 1553944.661405654)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303f595d50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303fad5d50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531074d8250>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b262430>, 1553944.955735158)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153043019d90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b261b70>, 1553944.52155445)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303f40ef10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304ba61630>, 1553943.945989304)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153100355750>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153103077310>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303f5d1550>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b659630>, 1553944.967303142)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531030743d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303f42fd90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b658d00>, 1553945.504419683)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303f42cf10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303f471dd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303f47e890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b658ec0>, 1553945.389515349)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303f470b10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531051adcc0>, 1553945.80802399)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303f47e750>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530566d8590>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b65be70>, 1553945.402768103)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fcce1c50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531067332d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b6599b0>, 1553945.673395494)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fd478150>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153042a59210>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b358c20>, 1553945.965326216)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303f43a690>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15305797fa10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304b75f990>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b3595c0>, 1553945.885011991)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15310683bcd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b261470>, 1553946.181664317)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304b75c890>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531017d4ed0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b3599b0>, 1553945.907419397)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fd0bd350>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153056d88650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b359400>, 1553946.10841319)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fcbfb6d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fc527610>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fd00c750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15310389c670>, 1553946.447578874)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fc524f90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15310389f930>, 1553946.3144775)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fd00e090>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b2604b0>, 1553945.116599515)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303f5d04d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303f364f90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b65b150>, 1553944.812134694)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303f366d50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153056ca3b50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15305723a750>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fcde25d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b35aeb0>, 1553946.728599369)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15310522ded0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fc3b1f90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b8577e0>, 1553946.642341795)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fc3b2d10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530566da250>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b856200>, 1553946.889181648)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153042fa1710>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531023f9b10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fc6f7650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531052454e0>, 1553946.90044875)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fce11710>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057e93c40>, 1553947.081146115)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fc6f41d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153105827cd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b8550f0>, 1553947.146458338)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153106c5fdd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fc78e9d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b856660>, 1553947.276188526)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fc78f050>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153105b60c10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fc95e910>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b65ba10>, 1553947.489345759)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153105b61c90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b855710>, 1553947.800698266)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fc95ec90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531030137d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304b75e550>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd1db000>, 1553947.924501458)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531030138d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056ee5860>, 1553947.805125566)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304b75d890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fcd2ed10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b358440>, 1553946.544160993)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303fb9cb90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303f4fb210>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153106c1dbd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b8559b0>, 1553946.701965048)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fcde31d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15310389ea50>, 1553946.194715312)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303f4f8dd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303fa9b7e0>, 1553948.486878483)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153106d01810>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b260ec0>, 1553946.338455694)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153106c1c550>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153105910790>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531063af310>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303fa9bbd0>, 1553947.508382959)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153056de9f50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153057af6450>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530578f0510>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303fa99da0>, 1553948.333349775)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153057af4590>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303fa9b2a0>, 1553948.843658741)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153057930150>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153057a40b90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303fbe15c0>, 1553948.426018168)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153057a40f50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153057aa12d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153105c0cad0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303fa988a0>, 1553948.908368804)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153102172f10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153106c7a690>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fc7fc550>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303fbe2120>, 1553948.847971846)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153106c7a010>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057f40f30>, 1553948.650212244)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fc7fea90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531076996d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303fbe1630>, 1553948.983907505)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15310769b850>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153057a34dd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303fb457f0>, 1553948.576421994)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fc4c7ad0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fe3ed690>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fd3b6ed0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303fa9af90>, 1553949.010872334)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303fbc4c90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303fb45550>, 1553948.880028686)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153056d7d890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153056de0f10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303fb47af0>, 1553949.159678562)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153056de36d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fc5f6010>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303fa994e0>, 1553948.006164667)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fc098310>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153105eee850>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b8574d0>, 1553948.47869473)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153057aa2590>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056ee62e0>, 1553949.594138664)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fc5f7810>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303fa9a0b0>, 1553947.818055093)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531051d8250>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153056dc7410>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057e92a50>, 1553949.565997466)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153056dc7490>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fccfb250>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531037c3930>, 1553949.197819898)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153057724f50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15310638ddd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fd689390>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531037c1940>, 1553949.525788673)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15310638eb50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531037c24a0>, 1553949.784921024)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fd44b950>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530579fb510>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fc5893d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531037c2c10>, 1553949.775529673)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fcc17990>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153057aa0a10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303fa9bb60>, 1553950.175336276)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153057aa1710>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304ba40510>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531037c3380>, 1553950.09914128)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304ba435d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fe409010>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153106645090>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531037c2cf0>, 1553950.000626336)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fe40b2d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531037c03d0>, 1553950.490094046)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153106647e10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fd4e58d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531037c0590>, 1553950.235884591)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fc227090>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304ba82810>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b855550>, 1553950.548613792)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153105c93dd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153106c79c10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303fb47000>, 1553950.741234888)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fd218bd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fc444ad0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531034ce490>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303fa98750>, 1553949.770818431)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fc588ad0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b7c0910>, 1553950.769046836)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fc446650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303fbe1cc0>, 1553951.145975767)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531034ccf90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153056dd6550>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b7c1da0>, 1553951.099822072)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153056dd73d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fceb8d10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304b639690>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303fb45e10>, 1553951.216472512)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fceb94d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b7c0590>, 1553951.244604578)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304b638d10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fced82d0>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b7c1ef0>, 1553951.233963924)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fced99d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304b85ae90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531037a2f90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b7c2e40>, 1553951.948327975)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304b859010>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b7c1b70>, 1553951.471433009)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531037a2750>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303f4f36d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303f52a5f0>, 1553951.485083005)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153056c05050>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303f4f2b90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531037504d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303f52b3f0>, 1553951.62243526)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303f4f2950>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303f52b690>, 1553951.783143738)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153103753250>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153103716410>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153101ed8710>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303f528e50>, 1553951.318794619)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fd1ad390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303fa40790>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531036728d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303f529f60>, 1553951.64929527)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fd16aa90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531037c2f20>, 1553952.755155898)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153103670cd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304ba11910>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b855240>, 1553951.768814187)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153103715a50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fc2e2b90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b855160>, 1553950.645793064)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304ba11c90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b7c1cc0>, 1553950.715798594)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304ba41610>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531035e18d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531035f0c90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b855320>, 1553952.17929472)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531035e1b10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531037c1ef0>, 1553952.214959364)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531035f0c50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153103664dd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303f5cd780>, 1553952.274662693)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153103665690>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15310377fa10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531034ca0d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303f5ce5f0>, 1553952.552362283)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15310360b150>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303f5cc280>, 1553952.105360009)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531034c9ad0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531035eaf90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303f5ce270>, 1553952.639522409)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531035e9450>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15310368f810>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531036dbed0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fe434e50>, 1553952.52599187)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153103686990>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303f5cec10>, 1553953.083457775)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531036da550>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531036de6d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303f5ccd70>, 1553952.939438041)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531036df850>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303f353950>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530429c7890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303f5ce660>, 1553953.249368789)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303f353810>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303f5cdbe0>, 1553953.572314766)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530429c5690>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153103883d10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b260e50>, 1553952.905791973)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153103882210>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531038414d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15305734cc20>, 1553953.710290904)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531038434d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153105273750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303f52acf0>, 1553953.107987816)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153103661c90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530429e0bd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531037d2750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15305734e890>, 1553953.674793332)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530429e3ed0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15305734dc50>, 1553953.275495751)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531037d0150>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153103998cd0>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15305734e900>, 1553953.706772495)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15310399a4d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15310373e7d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15305734da90>, 1553953.669148251)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15310373da10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15310397d410>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303fa989f0>, 1553954.170678979)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304ba38210>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153103912710>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531038d1190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15305734fb60>, 1553954.600894364)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531035bd5d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303f52b000>, 1553954.421968896)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153103822050>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531038cc850>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303f5cf150>, 1553954.14236508)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531038cd690>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153103ca0650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd42e9e0>, 1553954.574893058)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153103ca3090>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531035f8d90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531037c77d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd42cad0>, 1553954.354227412)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153056c3a110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153103c21190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fccb3460>, 1553954.224710236)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153103c21590>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153103bf7cd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531038cdf50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15305734ca60>, 1553955.044482709)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153103c67790>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153103da3b50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153103d9aa50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531036fb910>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd42fc40>, 1553954.401878317)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15310373c890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd42e740>, 1553954.699784812)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153103bf7ad0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153100bc2dd0>, 1553955.090940198)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153103bbf850>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153100bc3f50>, 1553954.746714549)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153103d9b410>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15305734f230>, 1553953.476783102)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531036fa210>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153103d60f90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fccb3070>, 1553954.899464991)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153103d62b10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153103bcc590>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153100bc3b60>, 1553955.332332699)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153103bcd350>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153103cce950>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bca8130>, 1553956.239580287)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153103ccf590>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153103d6c150>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153104432c50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153100bc2900>, 1553955.256031924)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153103d6d750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b7c2d60>, 1553955.416166812)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153103cb5890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153103d01dd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153104423390>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd5da660>, 1553955.360695153)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153103896bd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bca9f60>, 1553955.485692367)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531044231d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531044126d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bca9080>, 1553955.559923134)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153103cf7010>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153104392050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bcab070>, 1553956.107103797)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531038c3810>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15310366e190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153103d3acd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bcab540>, 1553955.993720708)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15310366f050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bca9b70>, 1553955.913452151)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153103d3a8d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531044e3e90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bcab7e0>, 1553956.927963615)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531044e2450>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15310453d310>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153104521110>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153105153a80>, 1553956.170825596)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15310453e710>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153105153150>, 1553956.833382478)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153104521d50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531044d5b90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bcabcb0>, 1553956.918832094)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531044d5010>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15310451f990>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153105150a60>, 1553956.521440897)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15310450ef90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531044924d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153104463cd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153105152430>, 1553956.697425088)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153104493450>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd42d1d0>, 1553957.065585727)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153104461b50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153104f36a10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153105150590>, 1553957.05629193)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153104f35610>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153103d21c50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153104ee2d90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531051518d0>, 1553957.10229157)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153103d42b50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153105152dd0>, 1553956.831174494)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153104ed54d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153104f1a050>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153104e85290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fccb27b0>, 1553957.323596529)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153103cf7810>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd5da7b0>, 1553957.239164074)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153104f4d350>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153104f07950>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15310502d890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b984830>, 1553957.31775798)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153104f07c10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b984ad0>, 1553957.477124609)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15310366ed10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153105010ad0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153105002310>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b986120>, 1553957.570232312)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153104e6aed0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153104393510>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153104f95b50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b985fd0>, 1553958.16103699)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153104fc52d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b987e70>, 1553958.593479002)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153104f97a10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153104f66c10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fccb2e40>, 1553958.806753935)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153104f65d50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153103d3b010>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153048b1f490>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153105151010>, 1553958.261874556)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153105003fd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b985860>, 1553957.895517411)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153103d3a6d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b9875b0>, 1553958.728611255)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153048b1d790>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153048af8fd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd5db930>, 1553958.693209602)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153048afb550>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15310450ee90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15305734e040>, 1553958.866625285)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531036d9e90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153103da0e10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153104f2a350>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bd10d70>, 1553958.873819818)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153103da1250>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bd10520>, 1553958.519620222)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153104f29a10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304bffef50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304bfcb650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531051508a0>, 1553958.840116428)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153048ac7c50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bd127b0>, 1553959.170538378)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304bfc8590>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304bf78450>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304bf6e6d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bd10830>, 1553958.963877074)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304bf78f50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bd10050>, 1553958.938699886)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304bf6ec10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304bf62750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bd13690>, 1553959.037648354)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531036dc7d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153048a36810>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153048a05750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bcaa350>, 1553959.089619793)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153048a35d50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bb21e10>, 1553959.557060451)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15310502d4d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304bfa4c90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530489ec410>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bb22200>, 1553959.306831449)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15310504db10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bb21400>, 1553959.991410211)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304bfef310>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304bf39d90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bb22b30>, 1553959.297672536)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304bf396d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153048985750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fe39f460>, 1553959.768601475)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530489873d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153104febd10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153048902e10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bb21b00>, 1553960.493753241)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304896fa50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530488ee6d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056d69a20>, 1553960.859567505)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531044802d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153048a51350>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd42e510>, 1553960.373366965)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153048a51110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530488c7050>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153048854d90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bca9fd0>, 1553960.826479501)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530488c5710>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056d68520>, 1553960.54489784)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153048857590>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531044cc650>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530566f8910>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056d68bb0>, 1553960.779243946)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531044ccd90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b9866d0>, 1553960.911033462)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530566f8a50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153048928b10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530566ca850>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056d6a510>, 1553960.962537205)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153048928390>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056d687c0>, 1553960.637271288)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530566c9bd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15305668e0d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530566a9dd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bd12970>, 1553960.775256404)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15305668ccd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153056663d10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bd13230>, 1553960.971399754)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153056663510>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153056647f50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303f475160>, 1553960.985645658)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153056645f50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304887bf50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304bec0e50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bb22580>, 1553960.066146782)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153048902d10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303f4765f0>, 1553961.013346303)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530566a8510>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303f4747c0>, 1553961.475541204)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304b548350>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303f477cb0>, 1553961.042862121)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304bec1510>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304be936d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303f474c90>, 1553961.974313147)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304be92350>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304bea0810>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304b60a9d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303f474910>, 1553960.966458009)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304bea2e90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304be24990>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303f4752b0>, 1553961.986348715)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304be24810>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304be11910>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304be04110>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bd12350>, 1553962.959622184)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304be07a50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304b2fa990>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304b484590>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304893eb90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd13c360>, 1553963.194527899)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304b2ba6d0>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304888b050>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304b2a1fd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd13ed60>, 1553963.326690341)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304888b410>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bcaaac0>, 1553962.908873403)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304b2a2950>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304beef550>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd759630>, 1553963.347851822)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304891c790>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304b522150>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bb220b0>, 1553962.863368932)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304b60ab50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056d69b00>, 1553963.077238285)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304be13d10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303f4756a0>, 1553962.340445097)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304b2fb610>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bd139a0>, 1553963.688555811)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304b4866d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303f4750f0>, 1553961.004611895)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304b522b10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd13dcc0>, 1553962.583042177)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304beed550>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304be95910>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531051529e0>, 1553963.082447594)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304b535cd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fe54ba10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd13c590>, 1553963.055605499)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153056686cd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153056d8c350>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531050f9ef0>, 1553963.174581868)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153056d8fa50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153102fcc590>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531050f95c0>, 1553963.794737484)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153102fcc350>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304be25010>\n",
      "Evaluating workflow:  56%|█████▌    | 1670/3000 [02:41<04:35,  4.84it/s]Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531050f91d0>, 1553963.946875323)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304be24550>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530573f89d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531050f8ec0>, 1553963.824576278)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153104e80310>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153057337a90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15305723d950>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531050f82f0>, 1553964.488525862)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153057335d90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b985940>, 1553964.374748671)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15305723d650>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304b273990>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fcee3460>, 1553964.648521311)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530488eeb90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304b3ba590>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304b663690>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304b44b910>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304b3fd8d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304b2154d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531050fbe00>, 1553962.947909191)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304b449d90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531050f9b70>, 1553962.994845371)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304b216410>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd13dda0>, 1553963.070768877)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304b36db50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bb231c0>, 1553964.586404609)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304b273b50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd13f690>, 1553962.419671329)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304b6620d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531050fb620>, 1553962.823829251)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304b3fdfd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530561424d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153105150910>, 1553965.154162021)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530561422d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153056150c90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530561384d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531050f8980>, 1553964.736341478)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15305730b590>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530561790d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fcee14e0>, 1553964.685624826)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530561727d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304b6c9790>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153056192350>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fc7ca970>, 1553964.972764424)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304b6dcad0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fc7cba80>, 1553965.080223096)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153056192150>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530561a31d0>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fca3f460>, 1553964.826017135)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530561a2f90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304b2c6810>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530561cd210>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fca3f690>, 1553965.061841807)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304b2c50d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530561dc290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531050f89f0>, 1553965.597916928)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530561dc610>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530561f8fd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bc2a510>, 1553965.509383861)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530561f8dd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530561fa6d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15305601ce90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bc2af90>, 1553965.853938655)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304b3f1890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bc2a5f0>, 1553965.371762571)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530561cde10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15305619a790>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bc2a4a0>, 1553965.683900307)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530561de950>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153056122650>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304b230890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fc7cbd20>, 1553964.835149959)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153056139a10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fca3d010>, 1553965.502809883)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530561cd010>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fc7c9550>, 1553964.561490509)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153056122d10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fc7c9080>, 1553964.635362943)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153056d8d4d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153057223310>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bc2a2e0>, 1553966.584264933)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15305612c390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153056171010>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bc290f0>, 1553966.631809205)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153056172250>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530560823d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153056097250>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bc2b930>, 1553966.337560822)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153056082150>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bc2bd20>, 1553966.580332121)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153056097110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153056043a90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530560bd9d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bc29630>, 1553966.817904366)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530560439d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531038d7700>, 1553966.614977087)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530560be090>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530560ca190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531038d5cc0>, 1553967.091354358)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530560333d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304bea0c10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bd13e00>, 1553966.775327764)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530428d0c10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15305612f550>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153055f154d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bc2aac0>, 1553966.546868271)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153056192190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153055f1d850>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531038d4f30>, 1553967.050776516)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153055f1df10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530560e6950>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530560f7f90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531038d7000>, 1553967.102116194)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530560e73d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531038d6eb0>, 1553967.152246298)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530560f7e90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153055f477d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531038d4130>, 1553966.904251476)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153055f47690>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153055f6fb90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531038d7c40>, 1553967.226019841)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153055f6f890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153055f7ce90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531038d6900>, 1553966.460401068)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153055f147d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531038d6970>, 1553967.359730652)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153055f7d590>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15305601e490>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153055f9b850>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531038d6b30>, 1553967.369385573)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153055f42690>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531050fbd90>, 1553967.508146147)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153055f98d50>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153055f7c890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bc2a740>, 1553967.450676487)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153055f7c410>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153055fd04d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153103c032a0>, 1553968.275495211)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304b567550>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153056074e10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153055e2e6d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531050fa2e0>, 1553968.75090747)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153055f7dad0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531050f83d0>, 1553968.400927608)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153055e2dd10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153055e323d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153103c029e0>, 1553968.432916734)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153055e32190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153055e32cd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153055e58310>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153103c02c80>, 1553968.520369203)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153055e32c50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153103d055c0>, 1553968.391346598)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153055e58950>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153055e6f610>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153103c02580>, 1553968.841120441)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153055e6f4d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153055e952d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153103d05cc0>, 1553968.45353033)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153055e950d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153055e11c50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bc28590>, 1553968.624984391)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153055e11b90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153055e73dd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153055fab550>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153103d04130>, 1553968.744670502)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153055ea7190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fc7cb070>, 1553967.244271076)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153055fab410>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153055e2fd90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531038d7310>, 1553969.841644317)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153055f5c110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153055e8c890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153055d07bd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153103d06740>, 1553968.975646802)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153055e8cb90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153055ed9990>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fca3ce50>, 1553969.379637858)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153055d07a90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153103d078c0>, 1553968.918894633)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304bedd390>\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  56%|█████▌    | 1681/3000 [02:41<01:36, 13.67it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  56%|█████▌    | 1684/3000 [02:41<01:41, 12.97it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  56%|█████▌    | 1687/3000 [02:42<02:51,  7.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  56%|█████▋    | 1689/3000 [02:42<02:57,  7.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  56%|█████▋    | 1693/3000 [02:43<02:33,  8.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  57%|█████▋    | 1698/3000 [02:43<01:47, 12.11it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  57%|█████▋    | 1703/3000 [02:43<01:23, 15.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  57%|█████▋    | 1705/3000 [02:43<01:26, 15.06it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  57%|█████▋    | 1707/3000 [02:44<02:14,  9.63it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  57%|█████▋    | 1709/3000 [02:44<02:47,  7.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  57%|█████▋    | 1711/3000 [02:45<02:54,  7.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  57%|█████▋    | 1717/3000 [02:45<01:52, 11.37it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  57%|█████▋    | 1722/3000 [02:45<01:26, 14.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  57%|█████▋    | 1724/3000 [02:45<01:32, 13.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  58%|█████▊    | 1726/3000 [02:46<01:44, 12.14it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  58%|█████▊    | 1728/3000 [02:46<02:14,  9.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  58%|█████▊    | 1730/3000 [02:46<02:51,  7.42it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  58%|█████▊    | 1735/3000 [02:46<01:39, 12.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  58%|█████▊    | 1737/3000 [02:47<02:08,  9.86it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  58%|█████▊    | 1742/3000 [02:47<01:36, 13.05it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  58%|█████▊    | 1744/3000 [02:47<01:38, 12.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  58%|█████▊    | 1746/3000 [02:48<02:01, 10.36it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  58%|█████▊    | 1748/3000 [02:48<01:59, 10.47it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  58%|█████▊    | 1750/3000 [02:48<02:33,  8.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  58%|█████▊    | 1754/3000 [02:48<02:20,  8.89it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  59%|█████▊    | 1758/3000 [02:49<01:55, 10.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  59%|█████▉    | 1764/3000 [02:49<01:39, 12.48it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  59%|█████▉    | 1766/3000 [02:49<01:48, 11.40it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  59%|█████▉    | 1768/3000 [02:50<01:45, 11.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  59%|█████▉    | 1770/3000 [02:50<02:27,  8.32it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  59%|█████▉    | 1772/3000 [02:50<02:13,  9.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  59%|█████▉    | 1774/3000 [02:50<02:01, 10.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  59%|█████▉    | 1777/3000 [02:51<02:21,  8.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  59%|█████▉    | 1781/3000 [02:51<01:42, 11.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  59%|█████▉    | 1783/3000 [02:51<01:51, 10.87it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  60%|█████▉    | 1785/3000 [02:51<01:57, 10.32it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  60%|█████▉    | 1789/3000 [02:52<01:51, 10.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  60%|█████▉    | 1794/3000 [02:52<01:37, 12.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  60%|█████▉    | 1796/3000 [02:52<01:47, 11.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  60%|█████▉    | 1798/3000 [02:53<02:02,  9.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  60%|██████    | 1800/3000 [02:53<01:58, 10.11it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  60%|██████    | 1802/3000 [02:53<02:06,  9.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  60%|██████    | 1806/3000 [02:54<02:12,  8.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  60%|██████    | 1810/3000 [02:54<02:21,  8.38it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  61%|██████    | 1816/3000 [02:54<01:33, 12.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  61%|██████    | 1818/3000 [02:55<01:26, 13.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  61%|██████    | 1821/3000 [02:55<01:23, 14.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  61%|██████    | 1823/3000 [02:55<01:58,  9.93it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  61%|██████    | 1825/3000 [02:55<02:00,  9.77it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  61%|██████    | 1829/3000 [02:56<02:07,  9.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  61%|██████    | 1833/3000 [02:56<01:40, 11.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  61%|██████    | 1836/3000 [02:56<01:25, 13.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  61%|██████▏   | 1839/3000 [02:57<01:29, 12.93it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  61%|██████▏   | 1842/3000 [02:57<01:37, 11.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  61%|██████▏   | 1844/3000 [02:57<02:04,  9.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  62%|██████▏   | 1846/3000 [02:57<02:11,  8.74it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  62%|██████▏   | 1847/3000 [02:58<02:10,  8.83it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  62%|██████▏   | 1852/3000 [02:58<01:43, 11.11it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  62%|██████▏   | 1854/3000 [02:58<01:35, 12.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  62%|██████▏   | 1856/3000 [02:58<01:44, 10.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  62%|██████▏   | 1860/3000 [02:59<01:55,  9.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  62%|██████▏   | 1864/3000 [02:59<01:39, 11.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  62%|██████▏   | 1869/3000 [03:00<01:48, 10.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  62%|██████▏   | 1874/3000 [03:00<01:40, 11.16it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  63%|██████▎   | 1876/3000 [03:01<02:24,  7.78it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  63%|██████▎   | 1881/3000 [03:01<01:57,  9.52it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  63%|██████▎   | 1885/3000 [03:01<01:40, 11.09it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  63%|██████▎   | 1888/3000 [03:01<01:18, 14.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  63%|██████▎   | 1890/3000 [03:02<01:52,  9.88it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  63%|██████▎   | 1892/3000 [03:02<01:39, 11.09it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  63%|██████▎   | 1896/3000 [03:02<01:44, 10.52it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  63%|██████▎   | 1898/3000 [03:02<01:48, 10.17it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  63%|██████▎   | 1900/3000 [03:03<01:38, 11.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  63%|██████▎   | 1902/3000 [03:03<01:45, 10.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  64%|██████▎   | 1906/3000 [03:03<01:34, 11.63it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  64%|██████▎   | 1908/3000 [03:03<01:42, 10.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  64%|██████▎   | 1912/3000 [03:04<02:00,  9.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  64%|██████▍   | 1918/3000 [03:04<01:45, 10.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  64%|██████▍   | 1920/3000 [03:05<01:46, 10.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  64%|██████▍   | 1925/3000 [03:05<01:27, 12.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  64%|██████▍   | 1928/3000 [03:05<01:22, 12.96it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  64%|██████▍   | 1930/3000 [03:06<01:38, 10.84it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  64%|██████▍   | 1932/3000 [03:06<01:40, 10.66it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  64%|██████▍   | 1934/3000 [03:06<01:45, 10.11it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  65%|██████▍   | 1936/3000 [03:06<01:40, 10.60it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  65%|██████▍   | 1938/3000 [03:06<01:40, 10.57it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  65%|██████▍   | 1940/3000 [03:07<01:43, 10.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  65%|██████▍   | 1944/3000 [03:07<01:40, 10.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  65%|██████▍   | 1948/3000 [03:07<01:32, 11.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  65%|██████▌   | 1950/3000 [03:07<01:28, 11.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  65%|██████▌   | 1952/3000 [03:08<01:42, 10.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  65%|██████▌   | 1954/3000 [03:08<01:33, 11.17it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  65%|██████▌   | 1956/3000 [03:08<01:48,  9.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  65%|██████▌   | 1960/3000 [03:09<01:47,  9.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  65%|██████▌   | 1964/3000 [03:09<01:51,  9.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  66%|██████▌   | 1966/3000 [03:09<01:51,  9.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  66%|██████▌   | 1969/3000 [03:09<01:38, 10.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  66%|██████▌   | 1973/3000 [03:10<01:32, 11.16it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  66%|██████▌   | 1975/3000 [03:10<02:00,  8.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  66%|██████▌   | 1980/3000 [03:10<01:25, 11.86it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  66%|██████▌   | 1982/3000 [03:11<01:38, 10.35it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  66%|██████▌   | 1985/3000 [03:11<01:28, 11.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  66%|██████▋   | 1989/3000 [03:11<01:36, 10.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  66%|██████▋   | 1991/3000 [03:12<01:29, 11.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  66%|██████▋   | 1993/3000 [03:12<01:29, 11.27it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  66%|██████▋   | 1995/3000 [03:12<01:31, 11.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  67%|██████▋   | 1999/3000 [03:12<01:40,  9.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  67%|██████▋   | 2001/3000 [03:13<01:57,  8.49it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  67%|██████▋   | 2002/3000 [03:13<02:00,  8.28it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  67%|██████▋   | 2006/3000 [03:13<01:44,  9.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  67%|██████▋   | 2009/3000 [03:13<01:32, 10.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  67%|██████▋   | 2011/3000 [03:14<01:23, 11.77it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  67%|██████▋   | 2018/3000 [03:14<01:20, 12.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  67%|██████▋   | 2020/3000 [03:14<01:32, 10.55it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  67%|██████▋   | 2024/3000 [03:15<01:37, 10.05it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  68%|██████▊   | 2027/3000 [03:15<01:48,  8.93it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  68%|██████▊   | 2030/3000 [03:15<01:34, 10.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  68%|██████▊   | 2035/3000 [03:16<01:34, 10.16it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  68%|██████▊   | 2040/3000 [03:17<01:32, 10.32it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  68%|██████▊   | 2042/3000 [03:17<01:44,  9.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  68%|██████▊   | 2046/3000 [03:17<01:23, 11.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  68%|██████▊   | 2050/3000 [03:17<01:11, 13.38it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  68%|██████▊   | 2052/3000 [03:18<01:30, 10.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  68%|██████▊   | 2054/3000 [03:18<01:24, 11.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  69%|██████▊   | 2056/3000 [03:18<02:03,  7.63it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  69%|██████▊   | 2058/3000 [03:18<01:53,  8.29it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  69%|██████▊   | 2060/3000 [03:19<01:51,  8.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  69%|██████▉   | 2064/3000 [03:19<01:33, 10.02it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  69%|██████▉   | 2066/3000 [03:19<01:24, 11.02it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  69%|██████▉   | 2068/3000 [03:19<01:21, 11.48it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  69%|██████▉   | 2074/3000 [03:20<01:13, 12.58it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  69%|██████▉   | 2076/3000 [03:20<01:31, 10.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  69%|██████▉   | 2078/3000 [03:20<01:44,  8.83it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  69%|██████▉   | 2081/3000 [03:21<01:48,  8.49it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  70%|██████▉   | 2086/3000 [03:21<01:24, 10.84it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  70%|██████▉   | 2088/3000 [03:21<01:20, 11.38it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  70%|██████▉   | 2091/3000 [03:22<01:15, 11.97it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  70%|██████▉   | 2093/3000 [03:22<01:22, 11.04it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  70%|██████▉   | 2095/3000 [03:22<01:39,  9.11it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  70%|███████   | 2100/3000 [03:23<01:38,  9.14it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  70%|███████   | 2106/3000 [03:23<01:02, 14.42it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  70%|███████   | 2109/3000 [03:24<01:36,  9.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  70%|███████   | 2115/3000 [03:24<01:08, 12.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  71%|███████   | 2117/3000 [03:24<01:48,  8.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  71%|███████   | 2119/3000 [03:25<01:45,  8.32it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  71%|███████   | 2123/3000 [03:25<01:37,  9.02it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  71%|███████   | 2125/3000 [03:25<01:26, 10.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  71%|███████   | 2132/3000 [03:26<01:04, 13.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  71%|███████   | 2134/3000 [03:26<01:27,  9.88it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  71%|███████   | 2136/3000 [03:26<01:22, 10.47it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  71%|███████▏  | 2138/3000 [03:26<01:32,  9.32it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  71%|███████▏  | 2140/3000 [03:27<01:55,  7.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  71%|███████▏  | 2142/3000 [03:27<01:51,  7.71it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  72%|███████▏  | 2147/3000 [03:27<01:11, 11.86it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  72%|███████▏  | 2152/3000 [03:28<01:11, 11.82it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  72%|███████▏  | 2154/3000 [03:28<01:08, 12.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  72%|███████▏  | 2156/3000 [03:28<01:17, 10.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  72%|███████▏  | 2158/3000 [03:29<01:41,  8.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  72%|███████▏  | 2160/3000 [03:29<01:33,  8.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  72%|███████▏  | 2163/3000 [03:29<01:52,  7.47it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  72%|███████▏  | 2165/3000 [03:29<01:37,  8.57it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  72%|███████▏  | 2169/3000 [03:30<01:25,  9.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  72%|███████▏  | 2174/3000 [03:30<01:11, 11.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  73%|███████▎  | 2176/3000 [03:30<01:09, 11.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  73%|███████▎  | 2182/3000 [03:31<01:20, 10.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  73%|███████▎  | 2184/3000 [03:32<01:39,  8.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  73%|███████▎  | 2186/3000 [03:32<01:30,  9.02it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  73%|███████▎  | 2191/3000 [03:32<01:03, 12.78it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  73%|███████▎  | 2195/3000 [03:32<01:24,  9.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  73%|███████▎  | 2197/3000 [03:33<01:25,  9.42it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  73%|███████▎  | 2199/3000 [03:33<01:36,  8.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  73%|███████▎  | 2202/3000 [03:33<01:21,  9.74it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  74%|███████▎  | 2208/3000 [03:34<01:13, 10.74it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  74%|███████▎  | 2212/3000 [03:34<01:07, 11.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  74%|███████▍  | 2214/3000 [03:34<01:01, 12.87it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  74%|███████▍  | 2216/3000 [03:34<01:05, 11.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  74%|███████▍  | 2220/3000 [03:35<01:26,  8.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  74%|███████▍  | 2222/3000 [03:35<01:24,  9.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  74%|███████▍  | 2224/3000 [03:35<01:12, 10.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  74%|███████▍  | 2226/3000 [03:36<01:19,  9.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  74%|███████▍  | 2231/3000 [03:36<01:09, 11.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  74%|███████▍  | 2234/3000 [03:36<01:16, 10.06it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  75%|███████▍  | 2236/3000 [03:37<01:22,  9.29it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  75%|███████▍  | 2239/3000 [03:37<01:35,  7.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  75%|███████▍  | 2241/3000 [03:37<01:36,  7.86it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  75%|███████▍  | 2244/3000 [03:38<01:17,  9.78it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  75%|███████▍  | 2248/3000 [03:38<00:56, 13.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  75%|███████▌  | 2251/3000 [03:38<01:13, 10.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  75%|███████▌  | 2254/3000 [03:38<01:09, 10.80it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  75%|███████▌  | 2256/3000 [03:39<01:36,  7.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153055d2c410>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531043d0c90>, 1553969.155628743)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153055d2c310>\n",
      "Evaluating workflow:  75%|███████▌  | 2257/3000 [03:40<02:37,  4.73it/s]Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153055d0df90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531043d0910>, 1553970.47729394)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153055d0dc90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153055d94190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531050fadd0>, 1553970.241353298)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153055d94bd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153055e2f050>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153055d43850>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531043d1da0>, 1553970.114481171)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153055e2f3d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531043d1b00>, 1553968.931800927)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153055d43710>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153055cef790>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153104f15390>, 1553971.649533802)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153055cef690>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153055b44890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153104f146e0>, 1553972.570185755)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153055b44b10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153055f1ebd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531038d70e0>, 1553972.348570311)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153055dab290>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153055b56ad0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531050347c0>, 1553972.418169127)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153055b56950>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153055bba290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531050363c0>, 1553972.554127504)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153055b63bd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153055bdbcd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153103d05fd0>, 1553973.200594483)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153055bdae50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153055a02510>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153105034de0>, 1553973.013035216)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153055a02690>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153055bc0890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153105037cb0>, 1553973.910880873)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153055caef10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153055a87710>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153105034750>, 1553973.530124616)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153055a87d10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15305592c990>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153104f14e50>, 1553974.730917675)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153055adc050>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530559654d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048a49470>, 1553974.422147946)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530559657d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153055aa5310>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531043d0830>, 1553973.444986471)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153055aa5110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530559edcd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531050342f0>, 1553975.451976051)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530559edc10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530559f3050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048a4ba80>, 1553975.044768993)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530559f2f10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153055735e10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bc2bd90>, 1553976.983748837)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153055735090>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153055886790>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530488baf20>, 1553977.719113975)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153055886750>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530557f9990>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530488b8830>, 1553978.125545653)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530557fa010>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15305551c250>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15305667a5f0>, 1553979.591378596)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15305551d290>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153055683f10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530488ba970>, 1553977.851855469)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153055683dd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153055551490>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b558520>, 1553980.104192579)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530555524d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530558c7290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b55aac0>, 1553980.69749752)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530558c7210>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15305546f750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b45e7b0>, 1553981.590728134)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15305546f4d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530554bced0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b45cd70>, 1553981.748219603)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530554b25d0>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15305538f910>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153103784f30>, 1553982.505261443)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530553dabd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530553c7f50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153103785fd0>, 1553983.106321877)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530553c7ed0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153055493610>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531037849f0>, 1553983.424966531)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153055387f50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530553a1e90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b45db00>, 1553983.917094063)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530553a1bd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15305528ebd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bba7d90>, 1553984.429624752)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153055121b90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530553d8810>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bba42f0>, 1553984.527543607)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530553da5d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530551b3150>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b45eac0>, 1553985.627394267)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530551b3890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153055179910>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b45fa80>, 1553984.305034731)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153055179fd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530550bbf10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530550925f0>, 1553986.707402043)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530550bbed0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530550cd050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153055092dd0>, 1553986.527142683)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15305509de50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153055068550>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153054ff7a90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530550bd590>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153054f7e0b0>, 1553988.346931607)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153055068950>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bba5710>, 1553985.710144751)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530550bd890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153054e01f90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153054f7e820>, 1553989.754415776)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153054e01d90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153054fc3250>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153054e254d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153054f7eac0>, 1553988.443102787)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153054fc26d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153054f7de10>, 1553988.52212293)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153054e25150>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153054e3cc10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153054f7f620>, 1553988.450842793)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153054e3d110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153054e2df10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153054f7fbd0>, 1553988.447607887)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153054e2e890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153054e72e10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153054e9fb90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bba7d20>, 1553988.582027124)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153054e72b10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153054ebf050>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153054ec8310>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153054e69f60>, 1553988.451976253)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153055197b90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153054e731d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153054eed950>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153054e6a120>, 1553988.360430466)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153054ec9e10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153055092820>, 1553988.448729932)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153054eed750>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153054f0a550>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153054d22190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153054e6a5f0>, 1553988.792190548)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153054f0b310>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530550918d0>, 1553988.437243285)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153054d17190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153054d40310>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153055093460>, 1553988.802051412)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153054d409d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153054d42290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153054f7f8c0>, 1553988.851372281)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153054e9fa50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530550937e0>, 1553988.452999666)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153054ebef10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153054e6b8c0>, 1553989.02660958)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153054d42790>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153054d26150>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153054d64c50>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153055d17f50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15305723e010>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153055d5ea10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153055d95550>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153055d22350>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153055dc0dd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153055dd1890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153055c0d650>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153055c26150>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153055d876d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153055c614d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153055c04490>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153055c40d10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153055cae1d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153055cc3cd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153055ce3dd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153055ced390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153055b2d350>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153055c42450>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153055c854d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153055c9cd10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153055c0e410>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153055b81110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153055b2a450>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153055bb4d90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153055e8ced0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153055bf0510>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153055a2fb90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153055a56110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153055a6dd50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153055a139d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153055ab84d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153055bc35d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153055ae4310>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153055904a50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153055adc190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15305593b890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15305595b750>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530559766d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530559871d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153055977e50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530559ca850>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15305581c390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530559d16d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15305582e310>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153055840610>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153055944650>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153055865b10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530558aac90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530558b4850>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530558c7250>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530558e9e10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530559bc950>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530558854d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530558a9190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153055719110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15305572db90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530557484d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153055705ad0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153055775d10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153055790e10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530557a9d90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530557c2d10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530557d9a50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153055816a90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530556280d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15305577e6d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530558e9c10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15305564c450>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153055659090>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530558eaa50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530556fbe10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530556f3a10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530557a9e50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530556e2d50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153055677210>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530556d1090>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530556d8b90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153055538a10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15305561eb50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153055592cd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15305555d9d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530555c25d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530555e1810>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530554166d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530554438d0>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153055465090>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530557aa890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530556f0410>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15305548a9d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153055491710>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530555c4750>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153055408150>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530554e1790>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153055323d10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530554e35d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153055344110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15305559e6d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153055344610>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530553a0f10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530553b60d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153055449550>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530553d8890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530553002d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15305530ce10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530555e3090>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530553f25d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153055219790>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153055539110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153055262850>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15305528cb10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153055345250>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530552d1150>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530552f4550>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153055117950>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530552a1750>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530552c9590>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530553b6390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153055169dd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530551968d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530551ba350>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153055246290>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153055029710>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15305503eed0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153055025110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15305505b890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153055069ad0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153055449a90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15305546f310>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15305528d510>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15305509c450>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530550d8d50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530550e4dd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153054f0a4d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153054f29750>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530551e7090>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153054f5ca50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153055158690>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153054f9fc90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153054fa7d50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153054f7e660>, 1553988.363172592)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153054ff7950>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153055197b10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153054f7c830>, 1553988.187870154)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530551971d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153054e6bbd0>, 1553989.714514443)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153054d48710>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153054d6c1a0>, 1553989.354780999)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153054d65150>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fc7c81a0>, 1553969.248319033)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153055d17e50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153103d048a0>, 1553969.150141303)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153055d0fb10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531043d27b0>, 1553969.6850173)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153055d5e7d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531043d3460>, 1553969.887962691)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153055d95410>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153103d07c40>, 1553970.282270678)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153055f54b90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531043d18d0>, 1553970.678882999)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153055ff9350>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531043d1b70>, 1553970.396870396)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153055dd1690>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153103d06e40>, 1553970.373878555)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153055c0d450>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153103c03460>, 1553971.209914421)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153055c25d90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531044ebaf0>, 1553971.065754361)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153055d87190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531044e86e0>, 1553970.839613408)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153055c61150>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531044eb1c0>, 1553970.66993084)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153055e8cd10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531044ea900>, 1553970.925374659)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153055c413d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531043d0b40>, 1553971.246903531)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153055cae890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531044e90f0>, 1553971.44910968)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153055cc3bd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153104f148a0>, 1553971.331418672)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153055ce3c90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153104f158d0>, 1553971.79562298)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153055ced850>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153104f17540>, 1553971.726335117)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153055b2d150>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153103c00a60>, 1553970.685365212)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153055c424d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531044e8b40>, 1553971.404288671)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153055c84bd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531038d60b0>, 1553971.306346278)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153055c9fc90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153103d04980>, 1553972.247554753)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153055cbed50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153104f15400>, 1553972.251191453)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153055b80f90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153104f17d90>, 1553972.736003933)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153055c42750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531050343d0>, 1553972.288199866)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153055bb4c50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531043d0f30>, 1553972.472247303)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153055ea43d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531044eb7e0>, 1553972.771473386)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153055bf0310>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153105036c10>, 1553973.328453876)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153055a2fa50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531044e8280>, 1553973.443800124)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153055a55fd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153105036350>, 1553973.578733506)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153055a6e1d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153105035e80>, 1553973.542401016)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153055a137d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048b254e0>, 1553973.546826969)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153055d70cd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048b25e10>, 1553973.978276734)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153055eda7d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153104f15a20>, 1553974.292210153)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153055ae4250>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048b24de0>, 1553974.349426714)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153055905190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153103c03f50>, 1553974.700361249)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153055a91850>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048b27000>, 1553974.17997594)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15305593b690>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048a48910>, 1553974.579433003)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15305595b610>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153105034670>, 1553974.46267468)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153055a96c50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048a49710>, 1553974.943703841)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153055987050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048a49240>, 1553974.871717146)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153055995e50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153104f14ec0>, 1553974.887121596)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530559ca650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048a4bc40>, 1553975.211815891)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15305581c710>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531044eb690>, 1553975.037584201)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153055a92210>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bf1e430>, 1553975.639488015)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15305590ee10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bf1e3c0>, 1553975.918880655)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530559d1290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531044eb150>, 1553976.010469057)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153055944590>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bf1d1d0>, 1553975.579168786)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153055865650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048b25d30>, 1553976.046834782)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530558ab110>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bf1fd20>, 1553976.062785524)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530558b4a10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bf1df60>, 1553975.998428179)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530558c7110>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048a48ad0>, 1553976.354448757)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530558ea4d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048a490f0>, 1553975.347526539)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530559b5890>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bf1def0>, 1553975.970087153)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153055886550>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048a48c20>, 1553976.036701605)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530558a8f90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048976890>, 1553976.823616868)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153055cc3e90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153103c024a0>, 1553976.81975147)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15305572dd90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048974e50>, 1553976.969049912)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153055ab87d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048977b60>, 1553977.166052133)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153055705bd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048977f50>, 1553976.877686965)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153055775b10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153103c00210>, 1553977.210811817)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153055790c10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048974b40>, 1553977.31596927)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530557a9c50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531050374d0>, 1553977.696335991)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530557c0450>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153105034ec0>, 1553977.821151324)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530557d9fd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153105035e10>, 1553978.057679845)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153055891ad0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530488b8130>, 1553977.517004862)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153055628750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048977ee0>, 1553977.2093113)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15305577c290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048b26ba0>, 1553977.582637863)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530558eab50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530488b8670>, 1553978.393729817)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15305564c2d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530488b9160>, 1553978.010327713)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153055659790>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530488ba2e0>, 1553978.60233995)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153055749350>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15305667ac10>, 1553978.699985112)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530556fbcd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15305667aa50>, 1553978.987600582)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530556f3950>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15305667acf0>, 1553978.90831977)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530557aae10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15305667be00>, 1553979.119901063)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530556e1f50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530488bb5b0>, 1553979.881927109)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530556770d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048974130>, 1553978.913317103)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530556d0f50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15305667a740>, 1553978.760613529)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530556d8a50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056679b70>, 1553979.350478525)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153055538f10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b55a580>, 1553979.520099085)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153055944510>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b559fd0>, 1553980.040124781)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153055593410>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b55bd20>, 1553979.913761554)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15305555e050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bf1c590>, 1553979.797342128)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530555c2e90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048b27b60>, 1553979.874470484)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530555e1610>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048a49a20>, 1553980.923768223)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530554175d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b558e50>, 1553980.651615882)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153055443f10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048a48bb0>, 1553980.842032421)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153055465710>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b45de10>, 1553980.929021858)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530557a9f10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b45fee0>, 1553981.468160437)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153055431f90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b45dcc0>, 1553981.145656817)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153055489210>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b45c910>, 1553982.142467955)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153055493a90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b5581a0>, 1553979.848122122)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530555c4a90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b55a200>, 1553980.316275233)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153055408710>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048975780>, 1553982.20090794)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530554e1c90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530488b89f0>, 1553982.112450663)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153055323bd0>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304be49160>, 1553981.854663136)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530554e3710>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304be48360>, 1553982.329208237)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153055474890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304be4b8c0>, 1553982.358861223)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153055346410>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b558d70>, 1553982.304574226)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530557f9fd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b559e80>, 1553982.359897359)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530553a0d90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b45e0b0>, 1553982.675875187)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530553b6810>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304be49400>, 1553983.034088782)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530553566d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b55a740>, 1553982.445141901)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530553d8510>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304be49e10>, 1553982.09824112)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153055300c90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15305667b930>, 1553981.485793508)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153055a02210>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530488b90f0>, 1553982.168273091)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15305530fed0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153103785470>, 1553983.123655647)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530553f2490>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153103785e80>, 1553983.499564488)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530552193d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153103785d30>, 1553983.420854031)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15305521af10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153103786cf0>, 1553983.64250544)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15305552e6d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b55b9a0>, 1553983.871846044)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15305528d890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153103787460>, 1553983.772728042)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530552a2dd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530488b9d30>, 1553983.975619547)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530552d0f50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bba73f0>, 1553984.261845663)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530552f4390>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bba7cb0>, 1553983.949766245)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153055117810>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304be48980>, 1553984.489404447)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530552a2a10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bba65f0>, 1553983.982528117)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530552c9450>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bba7a10>, 1553984.474638162)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530553b6f90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b55ad60>, 1553984.55011283)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153055169bd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153055184670>, 1553984.943495094)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530551966d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531037862e0>, 1553985.053520073)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530554054d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153055185b70>, 1553985.726235161)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530552462d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153103785160>, 1553985.066127683)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15305502b010>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530551875b0>, 1553985.787436573)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15305503cc90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530551862e0>, 1553985.280443716)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153055024090>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531037872a0>, 1553985.849607313)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15305505b750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153103785400>, 1553986.716630789)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15305506ba10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b45ee40>, 1553985.161905934)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153055449690>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153055185010>, 1553985.032676951)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530551dad50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153055090980>, 1553986.150503932)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153055884750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153055090bb0>, 1553985.998411233)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530551209d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304be49a20>, 1553986.659365656)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530550d8b50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b45faf0>, 1553986.852448157)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530550e4a10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530550927b0>, 1553986.754587141)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153054f0a2d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153055093000>, 1553986.444263293)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153054f29c50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b45fbd0>, 1553986.721880248)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530557b54d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531037850f0>, 1553987.166806698)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153054f5f450>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153054f7c670>, 1553987.215224021)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153054f73f90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153054f7cf30>, 1553986.94696519)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153054f9fb50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bba5b70>, 1553987.082989129)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153054fa7c10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530559047d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048975a20>, 1553976.325725542)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153055719210>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153055347850>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048b27c40>, 1553990.73839487)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153055346790>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153054df9450>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153054e774d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153054dd7010>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153104f150f0>, 1553990.627198143)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153054df92d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153104f16200>, 1553990.431827539)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153054f0b390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153054dc43d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153054db8e50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153054db38d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153103d05550>, 1553990.479401491)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153054f9ff10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153103d04600>, 1553990.597207372)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153054db8f90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153103d06890>, 1553990.845077112)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153054db2bd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153054c504d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153054c5c810>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531038d56a0>, 1553991.622551604)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15305521add0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531038d42f0>, 1553991.02310859)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153054c5cfd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153055263950>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15305576e6d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531038d5f60>, 1553990.786099801)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15305522de90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15305579b750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531038d49f0>, 1553990.764295957)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153055799e90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530556f29d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153055a759d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531038d50f0>, 1553991.254219091)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530556f19d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fca3e120>, 1553991.43749664)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153055a77d50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153055bb8610>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fca3fe70>, 1553991.834935194)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153055bb8490>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153054c257d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd75bd90>, 1553992.06808285)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153055a15bd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153055a6ec50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153055d67210>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153054c2ead0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153054dd42d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153054c3f110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530554a7f90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153104f15860>, 1553990.513742544)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153054dd7cd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530554f2b90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153055488c50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153054f7cc20>, 1553990.938161481)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15305576c310>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153103d06510>, 1553990.395867929)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153054c2f490>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153054e68280>, 1553992.194567529)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153055a6e4d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd75a820>, 1553992.455123769)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153055d66010>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bcaacf0>, 1553990.023680901)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153054c1add0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048a4a270>, 1553990.87816529)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530428aadd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048b260b0>, 1553990.076594952)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153054c3c7d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153105034590>, 1553990.242247781)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153054e01fd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153055187f50>, 1553990.330811919)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153054c6f1d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153055d22950>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153055184a60>, 1553992.337241817)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153054f83a10>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153055eeb510>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153055ec3d10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153054e6b000>, 1553992.33270881)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153055ee9010>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056d694e0>, 1553992.206312268)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153055e9f1d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153055e1ed50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153054e69ef0>, 1553992.57437793)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153055e1d5d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153055de3a50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153055fe6590>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056d68520>, 1553992.228346672)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153055de0b50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153055fb50d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153054e6b700>, 1553993.073840815)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153055fb6690>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153055fa1110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153055f5dbd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bca8600>, 1553992.450413556)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153055fa3c90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048a4b3f0>, 1553992.854113759)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153055f6fb50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15305521a110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530560be650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bcabe70>, 1553992.511423702)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15305500d650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056d695c0>, 1553992.686022012)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530560a5890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530560cbf10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bca9390>, 1553992.657738389)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530560c9350>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153055f15850>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bca8980>, 1553992.90799965)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153055f16650>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530560e5590>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153056041350>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bcabbd0>, 1553993.213381683)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153055a75e50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bcab540>, 1553993.407770656)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153056041450>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15305606b4d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153055fd36d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd5da890>, 1553992.782664381)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153055fe62d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b855ef0>, 1553993.642922668)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530560d7190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056d6a200>, 1553992.133169388)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153055fd0c10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153056005450>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153055ec0b50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd7592b0>, 1553993.86601479)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530560056d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd759630>, 1553994.125045606)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153055f54f50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530561cee10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153103d05cc0>, 1553993.821777121)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530561cf550>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530561ee590>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15305615d4d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b856b30>, 1553993.839505295)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530561ed810>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153054f7d390>, 1553993.876650794)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153055dc3cd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530561f9790>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b856ac0>, 1553993.872559791)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153056006ed0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530561b71d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b856660>, 1553994.459173947)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530561b4190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153055d71950>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153056105750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b854520>, 1553994.575874655)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153054c76790>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b35a350>, 1553993.936982189)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153056106d50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153056142d90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056d6a190>, 1553994.435627084)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153055e70dd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153055f98090>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b35b850>, 1553994.299314876)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304b4311d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304b3f1210>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304ba0f3d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b359d30>, 1553994.746663318)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304ba0e8d0>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304b4fa290>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304b3154d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b359710>, 1553994.976851068)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304b4f9f10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b854910>, 1553994.53896619)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304b284b50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304b36f950>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b359cc0>, 1553994.492360343)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304b246ad0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b35a430>, 1553994.682348173)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304b36db50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153056096a90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304b4e66d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b35add0>, 1553995.104528751)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530560945d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b35be00>, 1553995.040911859)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304b4e6850>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530566c8ed0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b35a2e0>, 1553995.226403458)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530566c9f90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153055d979d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b8549f0>, 1553995.402538089)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153055a6ead0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304be54b90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153056613110>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b3599b0>, 1553996.166847392)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304be54a90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153054e6aeb0>, 1553996.029721721)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530566127d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304be3d9d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153056621f10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b2601a0>, 1553995.939237576)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304be3ead0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b2607c0>, 1553995.615816699)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530566c2210>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153048987d50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153056105b50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b260210>, 1553995.761098545)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304be62210>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b263930>, 1553996.261955398)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153048879150>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153048955190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153103d07d20>, 1553996.405883265)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530489562d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153048962310>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153055d22a10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b855470>, 1553995.84682229)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153048960ad0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153105245cc0>, 1553996.341068893)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304b63e8d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530489d4ed0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b262200>, 1553996.607385203)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530489d5810>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153054c2d4d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304bf33890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd5daa50>, 1553996.357526683)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304b26bc90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fc520d70>, 1553996.418331155)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304bf32f90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304bf5c890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fc523af0>, 1553996.822478108)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304bf5c090>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153048aa4ed0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153048b08550>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b8552b0>, 1553996.707235871)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153048b08890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530566b54d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531051aef90>, 1553996.981867645)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304b4d8fd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153048afb310>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153104fd04d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531051ad320>, 1553997.081890971)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153048afaf90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531051ad4e0>, 1553997.143326235)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153104fd05d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153105020f10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b2609f0>, 1553997.768429989)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153105021210>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531044cd390>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531051af0e0>, 1553997.377874051)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531044cc250>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153104f6be50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531051ae040>, 1553997.5609509)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530488d1e90>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153104f642d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530428c10d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd758130>, 1553998.18275181)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304b646c50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057f41a20>, 1553997.660897711)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530428c2190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153104f22290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fc5218d0>, 1553997.720536618)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153104f223d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153104f1b750>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304bf32f10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304bf6e6d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153054e6aa50>, 1553996.763180807)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153048aa5cd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b2630e0>, 1553997.867850145)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153104ee3650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b8577e0>, 1553998.300380312)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304bf33b90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531051ae970>, 1553996.233052513)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153054dd0cd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153056104850>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056e79be0>, 1553998.308400822)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531044e0310>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153104449b90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531043a9690>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd6b7d90>, 1553998.337214443)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531044487d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd6b6ba0>, 1553998.849515008)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531043a8a10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531043e9150>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153103cf7290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd1d9da0>, 1553998.356718531)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531043e9ed0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd1da890>, 1553998.148526307)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153103cf7490>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153103d18050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056cccad0>, 1553998.506152948)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153103d1b210>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153103d99ad0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056ccd4e0>, 1553998.735399207)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153103d99810>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153048a88150>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531051aea50>, 1553998.955181464)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531044b5b50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153103bf6b90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153103bfc250>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fcd35ef0>, 1553998.671959488)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153103bf6f90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15310394b4d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fccb3850>, 1553999.053216829)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153103949510>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153103bcc050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531051adda0>, 1553998.973393666)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153104f7c290>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531038eee10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531051ad390>, 1553999.443348685)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153104fd13d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531038e8750>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530428b3b50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b262c10>, 1553999.597533393)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304b519ad0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd8d7770>, 1553999.545290232)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530428b15d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531037f4250>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153103850410>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b358d00>, 1553998.96891577)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153103bfc810>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd8d59b0>, 1553999.295711015)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531037f5990>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd8d7e70>, 1553999.643778119)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153103900950>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153103970550>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd8d44b0>, 1553999.797742698)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153103973ed0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153104462490>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153103821d90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b2627b0>, 1553999.676161024)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153104463050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd8d6900>, 1554000.072961391)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153104ea3fd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15310372d850>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531052469e0>, 1554000.18303065)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15310372c990>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153104549c50>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531036f2d50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd8d4bb0>, 1554000.366662399)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153103991090>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057f41b00>, 1553999.907498911)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531036f1ad0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153103896210>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd42d1d0>, 1554000.193688237)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153103896f50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153103d4be50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056dd9860>, 1554000.424082693)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153103d4bdd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153106644e10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bcab700>, 1554000.667509345)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fcb96510>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531035b3190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fc3c5fd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b2623c0>, 1554000.616061237)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531035cddd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd1d83d0>, 1554000.609682637)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153103cbf1d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15310379fa50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fe436900>, 1554000.662568794)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15310379ef10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303f3f3c90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153106662a50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fe436e40>, 1554001.646912415)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153056ceac10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fe4371c0>, 1554001.085930447)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153106662e50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531036f7f10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fe437a10>, 1554001.101928578)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531036f4d90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fceb9b10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303f5ce820>, 1554001.465922977)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fcebb950>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531059f9750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303f5ce040>, 1554001.64095747)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fc447d50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304b623650>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531034c2c90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303f5cf690>, 1554001.473627019)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fc465710>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303f5cf310>, 1554001.88913605)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531034c0310>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153056de1310>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd8d7620>, 1554001.964489731)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fd1e9fd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153056e7ddd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fd28bd10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303f5cd860>, 1554001.906515978)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fd288e90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153056dbfa50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303f5ceac0>, 1554002.072819525)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fc29f190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fd1d1810>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd505f60>, 1554002.122952557)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fd1d1010>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153057957890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531051ad9b0>, 1554001.910889009)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fc79d890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15310368c910>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd506120>, 1554002.064224713)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304ba38c50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15310765fb50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531076c4090>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057fdc520>, 1554002.273302994)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15310765f150>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057fdf5b0>, 1554002.159176426)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531076c5110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153101de50d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531051ac2f0>, 1554002.660873184)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153057933c10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153056d6e9d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153057aa2f10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303f5cda20>, 1554001.819015404)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fc1ad890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057fdfa80>, 1554002.311851751)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153106e68350>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057fdf620>, 1554002.69223551)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153057aa2f90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fc8cf790>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057fdf7e0>, 1554003.38221432)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fc8ced50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fc09b090>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531043d0280>, 1554002.875274356)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fc8659d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153107303610>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531043d1320>, 1554003.253162319)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153107303590>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304b75e810>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fcdc3d50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153103c01e10>, 1554003.298640671)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304b75ce90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153103c015c0>, 1554003.200454645)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153056ee04d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fcc485d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153106f3dfd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531043d3d90>, 1554003.693697706)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fcc49e50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153103c034d0>, 1554003.534212128)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153106f3ee50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153106fecb10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153103c02b30>, 1554003.650743945)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153106feda10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530428a0910>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bc291d0>, 1554003.83511094)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fcc03bd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153057841a10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15310218d3d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd1d9f60>, 1554003.819093092)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fc324710>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bc2ab30>, 1554003.543374402)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531062084d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303fb7d090>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bc2bbd0>, 1554003.800568476)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303fb7d710>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303fbdd190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bc299b0>, 1554004.119568659)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303fbdc4d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fca64a10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531052a86d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bc292b0>, 1554003.524986548)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153106bc3290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bc2b9a0>, 1554004.087999027)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531052ab250>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153057714f90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bc28590>, 1554003.643396079)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153057717490>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153056deb6d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531050f8830>, 1554004.345842673)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153056de8310>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303f5a2950>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531050fbbd0>, 1554004.712018014)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fc6f54d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530428bb010>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303faab7d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531050fa190>, 1554005.084948813)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530428bb7d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531050fb230>, 1554004.551600265)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303faab050>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303fa04b10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303fb1aa90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531050fa3c0>, 1554005.095262104)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303fbc4690>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531050f9f60>, 1554005.147264162)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303fa67d10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fc826c50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531050fb310>, 1554004.977502592)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530579f1a50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153056d53dd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b260de0>, 1554004.832150201)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153056d53210>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fd1e3990>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fc7c9fd0>, 1554005.381034234)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153057883c90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153042d62f90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fc7876d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fc7c9010>, 1554005.626389409)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153042d60ed0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fc7cbcb0>, 1554005.322389719)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530428a3510>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530431714d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd8d5320>, 1554006.300312075)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530431733d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153056df9ad0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531030774d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fccec3d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fc7c9be0>, 1554006.283419513)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153100b240d0>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bc296a0>, 1554005.584427083)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531030741d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bc28670>, 1554004.053845393)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fcced310>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15310308ee10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd13cd00>, 1554005.809289101)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15310308ead0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153100a470d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bc286e0>, 1554006.003145383)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153100a47e50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153104e69450>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153100970350>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd13dd30>, 1554006.197974761)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153104e69310>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153100a018d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057fdd710>, 1554006.553664244)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153100982190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153100a19550>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153054d6c3d0>, 1554006.637324002)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153100a180d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153100460b10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531043d2510>, 1554006.498091433)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153100460c50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153100347a10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153054d6e5f0>, 1554006.681492499)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531003441d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153100476490>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531050f95c0>, 1554006.61340997)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153100476210>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15305724ca50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15305723fcd0>\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153054d6d240>, 1554007.069847879)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15305724cf90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153054d6fee0>, 1554007.137656662)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15305723d150>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153057234e10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531050f81a0>, 1554007.154937675)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153057236450>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531003930d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153102ff7a90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153102fedf90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bc28ad0>, 1554006.490972862)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153100971c50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303f5ccad0>, 1554005.451887744)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153102fedfd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153054d6f150>, 1554007.2072533)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15310041e010>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bc2b5b0>, 1554005.552972769)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153102ff5010>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153103077450>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153054d6ed60>, 1554008.321164915)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153103077350>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531004f8b50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15310049ebd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153054d6eb30>, 1554007.522331289)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531004f9f90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fc7c9080>, 1554007.458154023)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15310049f510>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530573492d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303f5cfe00>, 1554007.578059813)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153057349850>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530572204d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303f476740>, 1554008.121491367)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530572238d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15305749a710>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153054d6ea50>, 1554008.046292696)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153057498fd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530573b0550>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fcee3460>, 1554007.959116291)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530573b0e10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530574a7410>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303f474440>, 1554008.707847013)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fc4b7010>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153057469d90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303f475fd0>, 1554008.125717683)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153100355a90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530574e14d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bb23a80>, 1554008.584450709)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530574e3490>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15305736c390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153102fcfe90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bb22d60>, 1554008.402653836)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15305736c210>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bb20c20>, 1554008.89724484)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153102fcf450>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15305737bdd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531038439d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fc7ca4a0>, 1554008.794920139)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153057379ed0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bb23230>, 1554008.783216827)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153057231310>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15305743bc10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531004f04d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bb20830>, 1554008.474433625)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15305743bb50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303f476430>, 1554006.959801355)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531009d0bd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15305732bad0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153054d6da90>, 1554008.853637297)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153057328050>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fd86d0d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153054d6ca60>, 1554009.271200642)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fd86ded0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15305723f390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153057238a50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fc7c8bb0>, 1554009.180260977)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fcde5450>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153054d6dda0>, 1554009.301099556)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15305723aa50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15305722b950>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fd814710>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153054d6de80>, 1554009.445248114)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153057407290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fe39c440>, 1554009.911598887)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fd814dd0>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fd807650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fe39c600>, 1554009.392605765)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fd805510>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fd77a5d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fd7acdd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bd11a90>, 1554009.737790501)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fd7ac150>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fd957810>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bd12eb0>, 1554009.821592912)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fd77a1d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fd954250>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fd8f15d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fcee1a20>, 1554009.687927151)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fd955d10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303f4750f0>, 1554010.034417545)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fd8f3c90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fd90e690>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bb22740>, 1554010.334800045)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fd90e310>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fd8a3d90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530573a56d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303f476120>, 1554010.173381423)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fd788610>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fe39f310>, 1554008.670277019)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530573a4f90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303f477460>, 1554010.110215431)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fd8a2410>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fd871f90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303f477620>, 1554010.54681369)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fd871850>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fda493d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b9873f0>, 1554010.779779881)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fda4b3d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fda47550>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fd9ecb50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b985080>, 1554010.394984525)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fda47090>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b985ef0>, 1554010.719308513)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fd9ed550>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fd8888d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fd9a7b90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b9878c0>, 1554010.612877584)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fd88bf50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b985d30>, 1554010.620173955)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fd9a41d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15305743acd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b985860>, 1554010.946311942)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15310016c850>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fdb1e1d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fdafe750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b986190>, 1554011.503911914)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fdafc750>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15305722ea50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fda8b490>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b9862e0>, 1554011.559439101)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153057239890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153054d6d160>, 1554011.485178378)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fda12c50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fdab0d50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153054d6fe70>, 1554011.317283904)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fdab3e10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fdc362d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fdc1d910>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b65ba10>, 1554011.568703636)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fdc1fed0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fdbd9210>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fc7c8210>, 1554010.969269647)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fdb1e650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fe39fbd0>, 1554011.29725456)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fd937190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b65b000>, 1554011.843801487)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fdbd9990>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fdbe3650>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fdb7bad0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b659550>, 1554011.851962088)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530574bec10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bd10670>, 1554012.787122247)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fdb79950>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fdb7ead0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b6586e0>, 1554012.265261265)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fdb7eb90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304287ce90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fdce6fd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b6588a0>, 1554012.015488592)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153057358050>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fdcbfdd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fdc804d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fdd27350>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153105150a60>, 1554012.446699983)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fdd268d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fdd39fd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531051511d0>, 1554012.080361309)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fdd385d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fd9a6a50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fe9d0c10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b985240>, 1554012.578030479)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fd9a7690>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bd11470>, 1554012.670235898)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fe9d1210>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fdd44710>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fdba05d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531051519b0>, 1554012.533824896)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fdd45250>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153105151710>, 1554012.464550549)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fdba2550>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fe92a510>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303f52a900>, 1554012.880090434)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fe92ae90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fdafe350>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530feae8110>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153105153ee0>, 1554012.216090093)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fdce7710>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b65a970>, 1554012.660132611)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fdcbf0d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bb21d30>, 1554013.063225759)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fdc83390>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303f52b380>, 1554013.135151867)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fdafcad0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303f529710>, 1554013.485154672)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530feae2310>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530feae9110>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303f52ba80>, 1554013.257597309)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fe92a490>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fd788250>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fea92010>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303f528440>, 1554013.649837606)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fea9a790>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303f52af90>, 1554013.349123114)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fea91e90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530ff620250>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fdb46f50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303f52a740>, 1554013.760094021)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fdb45190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fea4dd10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bd112b0>, 1554013.840636692)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fdd0bc90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530ff5c5d10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bd11be0>, 1554014.049976808)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530ff5c5fd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530ff5d6f90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530ff539010>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd7d2ba0>, 1554014.183030312)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530ff5d6f10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd7d1be0>, 1554013.868998145)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530ff538b10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530ff5aac10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530ff6f9610>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd7d34d0>, 1554014.433327958)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530ff5aab90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd7d0d00>, 1554014.275022609)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fdc8e6d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530ff57b210>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fea6a990>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd7d25f0>, 1554014.357391739)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fdc7c850>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd7d0b40>, 1554014.610085858)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fea69e10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530ff680450>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd7d10f0>, 1554014.736313922)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530ff682310>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530ff666390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531009d04d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303f529e10>, 1554013.592497134)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fea254d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd7d1710>, 1554014.927935843)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530ff666350>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd7d0590>, 1554014.657582028)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530ff648850>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530ffa086d0>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b7c1be0>, 1554014.777187937)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530ffa0a450>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530ff9fef90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b7c0ad0>, 1554014.85112718)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fe972b90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fdd02a10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b7c04b0>, 1554015.055499756)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fdd00a10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fea9a690>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530ff9684d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b7c1010>, 1554015.411354124)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153057421d90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b7c25f0>, 1554015.327149703)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530ff968d50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530ff665590>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b7c2580>, 1554015.268545409)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530ff93c6d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530ffb25e90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b7c01a0>, 1554015.598575332)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530ffb25f90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530ffafa110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530ffad14d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd7d38c0>, 1554015.187118105)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530ffaf9190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303f5296a0>, 1554015.615334785)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530ffad1610>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fea82790>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd9d4360>, 1554015.911253188)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fea821d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530ffb0bd50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530ffa40810>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd9d6190>, 1554016.728573662)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530ffa8e0d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd9d5400>, 1554016.51411105)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530ffa43a90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530ffc07bd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530ffbfacd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd9d7690>, 1554016.083733763)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530ffbfb650>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530ffa96c10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153105153e00>, 1554015.980185724)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530ffc06550>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd9d6c10>, 1554016.042938902)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530ffa94590>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530ffc1c8d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b659cc0>, 1554016.347391943)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530ff725490>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530ffb76dd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530ffb466d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b984de0>, 1554016.59449088)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530ffb76bd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd9d6740>, 1554017.313273925)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530ffb463d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530ffd27c90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531037c3f50>, 1554017.521058086)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530ffd26890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530ffb5b050>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530ffce7990>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd7d23c0>, 1554016.771225721)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530ffb58210>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530ffcb2010>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b7c2f20>, 1554016.398833473)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530ffcb0090>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530ffc8c250>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531037c12b0>, 1554016.95663758)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530ffc8db90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531002bd4d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531037c20b0>, 1554017.312037937)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531002bd690>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530ffaf9e50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530ffc81390>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531037c0bb0>, 1554017.874281804)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530ffaf93d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531037c1160>, 1554017.208800276)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530ff671ed0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530ffe16450>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531037c1860>, 1554017.154889681)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530ffe16dd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153100197d10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303fb44360>, 1554018.047079789)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153100196b50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530ffbbc4d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303fb45b70>, 1554017.878904692)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530ffa29c90>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530ffdceb50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530ffdc1050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd9d5860>, 1554016.81066416)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530ffce7c50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd9d63c0>, 1554018.003992739)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530ffdd9050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303fb44f30>, 1554017.688107279)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530ffdc1990>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530ffea1a10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303fb457f0>, 1554018.304899635)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530ffdb66d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530ff547090>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531005266d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303fb46350>, 1554017.973697149)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530ffb6ba10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303fb45160>, 1554018.477974965)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530ffd5e390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15310095f490>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531002f5550>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153105150bb0>, 1554018.342323166)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15310095e0d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304ba602f0>, 1554018.264605247)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531002f5650>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530ffa5f190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd9d6890>, 1554018.909135692)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531001ae410>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530ffd3e950>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fff0d190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304ba63310>, 1554018.95205895)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530ffcca950>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304ba606e0>, 1554018.711549168)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fff0d910>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530ffc59250>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304ba63a10>, 1554018.860976585)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530ffc5a110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153100193f50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304ba615c0>, 1554019.417748981)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153100193ed0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530ffe8ba50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304ba60360>, 1554018.717374375)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530ffe8bed0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530ffef6550>\n",
      "Evaluating workflow:  75%|███████▌  | 2261/3000 [03:40<01:47,  6.90it/s]Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531001fc050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304ba63690>, 1554019.227070393)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fd94f0d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304ba61e10>, 1554019.048335246)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530ffef6010>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530ffec0050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd221da0>, 1554019.192471136)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530ffec16d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531003e9bd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b987150>, 1554017.962169588)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531003e8dd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530ffaf9610>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304ba60bb0>, 1554019.571711587)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530ffb4c910>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153054c1e950>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153054c96950>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd2207c0>, 1554020.933482301)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530ff976a50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153054c9c050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd223c40>, 1554019.842565184)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153054c9ead0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15305458c290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd223ee0>, 1554019.394644916)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153054c9ff10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530545a2dd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530545bcfd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd222f20>, 1554020.112285588)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530ffb083d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303fb45630>, 1554019.690182076)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530545bcdd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530545d0cd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd344830>, 1554020.191780907)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530545d0b90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530ffe72810>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530545e3290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd347230>, 1554020.620153047)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530ffe73410>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd345a90>, 1554020.342721893)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530545e3150>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153054411390>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057e900c0>, 1554020.960576849)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153054412490>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15305442ecd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531037c2c80>, 1554020.326823785)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15305442eb90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530ff9fc590>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057e91e80>, 1554020.536164417)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530ff9fffd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15305444dcd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153054464810>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057e93850>, 1554020.701472364)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15305444dfd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530544738d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd220c20>, 1554019.871635342)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530ffc78c90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bd8c750>, 1554020.794752864)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153054462190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bd8eba0>, 1554021.149201782)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153054473790>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530544906d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530544b1410>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531037c0ad0>, 1554021.058599585)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153054490690>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b7c29e0>, 1554021.144614689)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530544b1710>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530544c8410>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304ba61e80>, 1554021.100803189)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530544c8b10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530544ca350>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd222510>, 1554021.611798962)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530544ca650>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530ffd6b9d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530544fbf90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd9d5240>, 1554021.115131257)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fead3310>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bd8f2a0>, 1554021.57451144)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530544fbe90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15305430f1d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fdaeaac0>, 1554021.446617657)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15305430f090>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530544660d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd344f30>, 1554021.620988948)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15305431f390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530545d1c90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fdaea510>, 1554021.884600094)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530545d1dd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15305432ffd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fdae80c0>, 1554022.370969692)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15305432fed0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15305436de50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153054376b50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fdaeb070>, 1554022.184702505)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15305436e090>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fdae8de0>, 1554022.199031268)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153054376950>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530543654d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fdae86e0>, 1554022.275276543)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530543659d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530544814d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531001ccad0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304ba60590>, 1554020.78743413)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530ffdc3ed0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fdae9940>, 1554022.336684446)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530ffdc0b50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530543678d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530543eced0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd344a60>, 1554022.759278042)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fff15910>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530550911d0>, 1554022.950836462)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530543eccd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530542142d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057e918d0>, 1554022.738038012)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153054214950>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530543a6290>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153054225150>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057e93930>, 1554023.587009606)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530543a5a10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fdbf8c90>, 1554023.920219379)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153054225890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530542322d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fdbf9da0>, 1554023.908993867)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530542320d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530543b7a50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153054248690>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fdbfbaf0>, 1554022.824141177)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530543b7a90>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fdbf94e0>, 1554024.200950368)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153054248d90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153054261f10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fdbf99b0>, 1554023.047962158)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153054261d90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15305428cf10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fdbf8bb0>, 1554023.419045025)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15305448add0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153054491fd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fdbfb770>, 1554023.358943637)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153054492710>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153054c940d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530542b8590>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fdaebd20>, 1554023.333360248)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530542a0810>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fdbfa3c0>, 1554023.500894194)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530542b8390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530545be4d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fdbfbb60>, 1554024.00593548)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153054281b90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530543c7f50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057e90210>, 1554022.380844187)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530543c7e50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530543a7290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fdcfd160>, 1554024.331105533)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153054481410>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15305410a1d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153054122790>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fdcff070>, 1554024.47164824)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15305410a0d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fdaeb770>, 1554024.422569996)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153054121210>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530544eefd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153054141050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fdaebee0>, 1554024.726560706)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15305412ab50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fdcff380>, 1554024.590268459)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153054140e50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530543eecd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153054185f50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fdcfc0c0>, 1554024.555834968)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530543ee2d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fdcff1c0>, 1554024.605693495)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153054185e10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15305419db50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fdcfcc20>, 1554024.827090894)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15305419d950>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153054232bd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fdcfd320>, 1554025.861997477)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15305440a190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153054187fd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fdcfdef0>, 1554024.838327737)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153054187e50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530541e4610>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fe9b38c0>, 1554024.963494489)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530541e4410>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530541c7050>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153049b00b50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fe9b02f0>, 1554025.133850735)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530541c7950>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fe9b0130>, 1554025.128366035)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153049b00a10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153049b183d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fdcfed60>, 1554025.275856581)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153049b18a10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530541c5050>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153054464d50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fe9b0670>, 1554025.593613738)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530541c5110>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fdcfe270>, 1554024.042864296)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530543a5810>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153054226190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153049b54a10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fe9b00c0>, 1554026.503435898)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153054226290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fe9b3ee0>, 1554025.73938155)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153049b548d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153049b641d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fe9b0600>, 1554025.931382731)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530543a4e90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153049b3af10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fe9b23c0>, 1554026.161264504)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153049b3ae50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530541bb210>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fdcffcb0>, 1554026.381297708)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153049b8a8d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153054165bd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153049bc0510>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fdaeb3f0>, 1554026.728084005)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153049b9f490>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530feabd400>, 1554025.952097812)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153049bc0bd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153049b3a910>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057e92e40>, 1554026.232192805)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530542d22d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153049b51310>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153049bf3c50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fdcfc210>, 1554026.642761951)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153049bd8ed0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153049bfac10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530feabdb70>, 1554027.54006726)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153049bfaa10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153049a0ae90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530feabd160>, 1554026.39660635)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153049a0b210>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153049a21450>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153049a2d2d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fdcfd0f0>, 1554026.833717038)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153049a21310>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530feabed60>, 1554026.593915018)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153049a2d190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153049a447d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fdbfbe00>, 1554027.063441394)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153049a44690>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153049a5a050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530feabff50>, 1554026.886749733)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153049bf3b10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fdae9ef0>, 1554026.878103503)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153049a5a5d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153049a3c950>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530feabdc50>, 1554027.141737149)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153049a64090>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153049a9b050>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153049a7d510>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304ba601a0>, 1554027.313484827)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153049a9aed0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530ff5cb700>, 1554026.799987054)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153049a72190>\n",
      "Evaluating workflow:  76%|███████▌  | 2269/3000 [03:40<00:53, 13.60it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  76%|███████▌  | 2272/3000 [03:40<00:52, 13.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  76%|███████▌  | 2275/3000 [03:41<01:16,  9.47it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  76%|███████▌  | 2277/3000 [03:42<01:46,  6.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  76%|███████▌  | 2280/3000 [03:42<01:32,  7.78it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  76%|███████▌  | 2286/3000 [03:42<01:03, 11.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  76%|███████▋  | 2288/3000 [03:42<01:07, 10.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  76%|███████▋  | 2290/3000 [03:42<01:05, 10.87it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  76%|███████▋  | 2292/3000 [03:43<01:06, 10.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  76%|███████▋  | 2294/3000 [03:43<01:25,  8.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  77%|███████▋  | 2296/3000 [03:43<01:45,  6.66it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  77%|███████▋  | 2299/3000 [03:44<01:22,  8.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  77%|███████▋  | 2304/3000 [03:44<00:57, 12.09it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  77%|███████▋  | 2306/3000 [03:44<00:57, 12.05it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  77%|███████▋  | 2308/3000 [03:44<01:02, 10.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  77%|███████▋  | 2310/3000 [03:45<00:55, 12.37it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  77%|███████▋  | 2312/3000 [03:45<01:03, 10.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  77%|███████▋  | 2314/3000 [03:45<01:14,  9.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  77%|███████▋  | 2316/3000 [03:45<01:15,  9.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  77%|███████▋  | 2319/3000 [03:46<01:19,  8.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  77%|███████▋  | 2321/3000 [03:46<01:13,  9.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  78%|███████▊  | 2325/3000 [03:46<01:00, 11.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  78%|███████▊  | 2327/3000 [03:46<00:57, 11.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  78%|███████▊  | 2329/3000 [03:47<01:09,  9.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  78%|███████▊  | 2333/3000 [03:47<01:01, 10.87it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  78%|███████▊  | 2335/3000 [03:47<00:54, 12.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  78%|███████▊  | 2337/3000 [03:47<01:02, 10.58it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  78%|███████▊  | 2339/3000 [03:48<01:17,  8.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  78%|███████▊  | 2343/3000 [03:48<01:14,  8.87it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  78%|███████▊  | 2348/3000 [03:48<00:51, 12.57it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  78%|███████▊  | 2353/3000 [03:49<00:58, 11.11it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  78%|███████▊  | 2355/3000 [03:49<00:56, 11.37it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  79%|███████▊  | 2357/3000 [03:49<01:05,  9.78it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  79%|███████▊  | 2361/3000 [03:50<01:09,  9.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  79%|███████▉  | 2366/3000 [03:50<00:44, 14.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  79%|███████▉  | 2370/3000 [03:50<00:48, 13.06it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metricsmetrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      " {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  79%|███████▉  | 2372/3000 [03:51<01:18,  8.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  79%|███████▉  | 2374/3000 [03:51<01:12,  8.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  79%|███████▉  | 2376/3000 [03:51<01:09,  9.04it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  79%|███████▉  | 2380/3000 [03:52<01:00, 10.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  79%|███████▉  | 2384/3000 [03:52<00:57, 10.74it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  80%|███████▉  | 2388/3000 [03:52<00:48, 12.60it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  80%|███████▉  | 2390/3000 [03:52<00:50, 12.11it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  80%|███████▉  | 2392/3000 [03:53<01:03,  9.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  80%|███████▉  | 2394/3000 [03:53<01:17,  7.83it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  80%|███████▉  | 2398/3000 [03:54<01:04,  9.38it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  80%|████████  | 2400/3000 [03:54<00:58, 10.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  80%|████████  | 2405/3000 [03:54<00:47, 12.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  80%|████████  | 2410/3000 [03:54<00:39, 15.06it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  80%|████████  | 2412/3000 [03:55<01:03,  9.32it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  81%|████████  | 2416/3000 [03:55<00:53, 11.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  81%|████████  | 2420/3000 [03:56<01:10,  8.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  81%|████████  | 2422/3000 [03:56<00:59,  9.63it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  81%|████████  | 2426/3000 [03:56<00:54, 10.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  81%|████████  | 2431/3000 [03:57<00:44, 12.74it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  81%|████████  | 2433/3000 [03:57<00:43, 13.02it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  81%|████████  | 2435/3000 [03:57<00:50, 11.14it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  81%|████████  | 2437/3000 [03:57<01:03,  8.92it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  81%|████████▏ | 2439/3000 [03:58<01:09,  8.04it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  81%|████████▏ | 2442/3000 [03:58<01:04,  8.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  82%|████████▏ | 2446/3000 [03:58<00:57,  9.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  82%|████████▏ | 2449/3000 [03:58<00:43, 12.61it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  82%|████████▏ | 2452/3000 [03:59<00:38, 14.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  82%|████████▏ | 2456/3000 [03:59<00:52, 10.32it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  82%|████████▏ | 2460/3000 [04:00<00:50, 10.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  82%|████████▏ | 2465/3000 [04:00<00:54,  9.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  82%|████████▏ | 2467/3000 [04:00<00:48, 11.07it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  82%|████████▏ | 2469/3000 [04:00<00:51, 10.38it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  82%|████████▏ | 2474/3000 [04:01<00:45, 11.58it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  83%|████████▎ | 2476/3000 [04:01<00:51, 10.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  83%|████████▎ | 2483/3000 [04:02<00:47, 10.97it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  83%|████████▎ | 2487/3000 [04:02<00:45, 11.16it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  83%|████████▎ | 2489/3000 [04:02<00:52,  9.77it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  83%|████████▎ | 2494/3000 [04:03<00:43, 11.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  83%|████████▎ | 2496/3000 [04:03<00:41, 12.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  83%|████████▎ | 2500/3000 [04:04<00:48, 10.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  83%|████████▎ | 2502/3000 [04:04<00:45, 10.93it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  83%|████████▎ | 2504/3000 [04:04<00:47, 10.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  84%|████████▎ | 2508/3000 [04:04<00:52,  9.38it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  84%|████████▍ | 2513/3000 [04:05<00:49,  9.83it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  84%|████████▍ | 2516/3000 [04:05<00:40, 11.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  84%|████████▍ | 2520/3000 [04:05<00:37, 12.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  84%|████████▍ | 2522/3000 [04:06<00:37, 12.84it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  84%|████████▍ | 2524/3000 [04:06<00:41, 11.60it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  84%|████████▍ | 2526/3000 [04:06<01:09,  6.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  84%|████████▍ | 2529/3000 [04:07<00:53,  8.84it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  84%|████████▍ | 2533/3000 [04:07<00:56,  8.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  85%|████████▍ | 2538/3000 [04:07<00:37, 12.46it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  85%|████████▍ | 2541/3000 [04:08<00:35, 12.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  85%|████████▍ | 2543/3000 [04:08<00:39, 11.67it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  85%|████████▍ | 2545/3000 [04:08<00:56,  8.04it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  85%|████████▍ | 2548/3000 [04:09<00:56,  7.96it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  85%|████████▌ | 2551/3000 [04:09<00:46,  9.71it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  85%|████████▌ | 2555/3000 [04:09<00:38, 11.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  85%|████████▌ | 2560/3000 [04:09<00:28, 15.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  85%|████████▌ | 2563/3000 [04:10<00:25, 16.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  86%|████████▌ | 2565/3000 [04:10<01:00,  7.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  86%|████████▌ | 2569/3000 [04:11<00:47,  9.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  86%|████████▌ | 2571/3000 [04:11<00:48,  8.78it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  86%|████████▌ | 2574/3000 [04:11<00:35, 11.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  86%|████████▌ | 2582/3000 [04:11<00:25, 16.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  86%|████████▌ | 2587/3000 [04:12<00:46,  8.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  86%|████████▋ | 2589/3000 [04:13<00:49,  8.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  87%|████████▋ | 2596/3000 [04:13<00:32, 12.32it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  87%|████████▋ | 2600/3000 [04:13<00:27, 14.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  87%|████████▋ | 2602/3000 [04:14<00:31, 12.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  87%|████████▋ | 2605/3000 [04:14<00:43,  9.06it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  87%|████████▋ | 2611/3000 [04:15<00:44,  8.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  87%|████████▋ | 2617/3000 [04:15<00:29, 13.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  87%|████████▋ | 2621/3000 [04:15<00:22, 16.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  87%|████████▋ | 2624/3000 [04:16<00:23, 16.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  88%|████████▊ | 2627/3000 [04:16<00:46,  8.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  88%|████████▊ | 2629/3000 [04:17<00:50,  7.32it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  88%|████████▊ | 2633/3000 [04:17<00:35, 10.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  88%|████████▊ | 2638/3000 [04:17<00:30, 11.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  88%|████████▊ | 2643/3000 [04:17<00:23, 15.46it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  88%|████████▊ | 2645/3000 [04:18<00:38,  9.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  88%|████████▊ | 2647/3000 [04:18<00:41,  8.52it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  88%|████████▊ | 2651/3000 [04:19<00:38,  9.11it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  88%|████████▊ | 2653/3000 [04:19<00:38,  9.02it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  89%|████████▊ | 2658/3000 [04:19<00:27, 12.55it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  89%|████████▊ | 2661/3000 [04:19<00:21, 15.57it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  89%|████████▉ | 2663/3000 [04:20<00:24, 13.72it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  89%|████████▉ | 2667/3000 [04:20<00:36,  9.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  89%|████████▉ | 2670/3000 [04:21<00:36,  9.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  89%|████████▉ | 2675/3000 [04:21<00:29, 11.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  89%|████████▉ | 2679/3000 [04:21<00:21, 15.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  89%|████████▉ | 2681/3000 [04:21<00:29, 10.88it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  90%|████████▉ | 2685/3000 [04:22<00:32,  9.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  90%|████████▉ | 2687/3000 [04:22<00:35,  8.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  90%|████████▉ | 2689/3000 [04:23<00:40,  7.63it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  90%|████████▉ | 2697/3000 [04:23<00:21, 14.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  90%|████████▉ | 2699/3000 [04:23<00:27, 10.77it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  90%|█████████ | 2701/3000 [04:23<00:25, 11.66it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  90%|█████████ | 2703/3000 [04:24<00:29, 10.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  90%|█████████ | 2705/3000 [04:24<00:38,  7.61it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  90%|█████████ | 2709/3000 [04:24<00:33,  8.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  90%|█████████ | 2713/3000 [04:25<00:25, 11.09it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  91%|█████████ | 2716/3000 [04:25<00:21, 13.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  91%|█████████ | 2722/3000 [04:25<00:21, 13.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  91%|█████████ | 2724/3000 [04:25<00:23, 11.86it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  91%|█████████ | 2730/3000 [04:26<00:27,  9.84it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  91%|█████████ | 2732/3000 [04:26<00:27,  9.86it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  91%|█████████ | 2734/3000 [04:27<00:28,  9.47it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  91%|█████████▏| 2740/3000 [04:27<00:19, 13.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  91%|█████████▏| 2742/3000 [04:27<00:20, 12.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  91%|█████████▏| 2744/3000 [04:28<00:28,  9.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  92%|█████████▏| 2749/3000 [04:28<00:25,  9.96it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  92%|█████████▏| 2751/3000 [04:28<00:27,  8.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  92%|█████████▏| 2755/3000 [04:29<00:22, 10.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  92%|█████████▏| 2759/3000 [04:29<00:15, 15.86it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  92%|█████████▏| 2762/3000 [04:29<00:20, 11.49it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  92%|█████████▏| 2764/3000 [04:29<00:21, 11.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  92%|█████████▏| 2766/3000 [04:30<00:24,  9.74it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  92%|█████████▏| 2768/3000 [04:30<00:24,  9.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  92%|█████████▏| 2772/3000 [04:30<00:22, 10.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  93%|█████████▎| 2776/3000 [04:31<00:20, 11.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  93%|█████████▎| 2779/3000 [04:31<00:16, 13.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  93%|█████████▎| 2781/3000 [04:31<00:17, 12.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  93%|█████████▎| 2785/3000 [04:31<00:19, 10.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  93%|█████████▎| 2787/3000 [04:32<00:24,  8.80it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  93%|█████████▎| 2789/3000 [04:32<00:23,  8.86it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  93%|█████████▎| 2794/3000 [04:32<00:15, 13.37it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  93%|█████████▎| 2796/3000 [04:32<00:15, 13.02it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  93%|█████████▎| 2798/3000 [04:32<00:15, 13.35it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  93%|█████████▎| 2801/3000 [04:33<00:13, 14.28it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  93%|█████████▎| 2803/3000 [04:33<00:18, 10.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  94%|█████████▎| 2805/3000 [04:33<00:18, 10.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  94%|█████████▎| 2809/3000 [04:34<00:18, 10.16it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  94%|█████████▎| 2811/3000 [04:34<00:16, 11.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  94%|█████████▍| 2815/3000 [04:34<00:17, 10.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  94%|█████████▍| 2817/3000 [04:34<00:16, 10.80it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  94%|█████████▍| 2821/3000 [04:35<00:14, 12.77it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  94%|█████████▍| 2823/3000 [04:35<00:20,  8.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  94%|█████████▍| 2825/3000 [04:35<00:22,  7.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  94%|█████████▍| 2829/3000 [04:36<00:17,  9.60it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  94%|█████████▍| 2831/3000 [04:36<00:20,  8.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  94%|█████████▍| 2835/3000 [04:36<00:13, 12.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  95%|█████████▍| 2839/3000 [04:36<00:12, 12.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  95%|█████████▍| 2841/3000 [04:37<00:12, 12.67it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  95%|█████████▍| 2845/3000 [04:37<00:17,  9.07it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  95%|█████████▌| 2850/3000 [04:38<00:15,  9.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  95%|█████████▌| 2853/3000 [04:38<00:12, 11.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  95%|█████████▌| 2856/3000 [04:38<00:10, 13.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  95%|█████████▌| 2861/3000 [04:39<00:11, 12.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  95%|█████████▌| 2863/3000 [04:39<00:14,  9.67it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  96%|█████████▌| 2865/3000 [04:39<00:12, 10.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  96%|█████████▌| 2870/3000 [04:39<00:11, 11.46it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  96%|█████████▌| 2872/3000 [04:40<00:10, 12.71it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  96%|█████████▌| 2876/3000 [04:40<00:11, 11.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  96%|█████████▌| 2879/3000 [04:40<00:09, 13.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153049ae06d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530ff5c9240>, 1554027.845202448)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153049ae0910>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153049b8a950>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fdbf8b40>, 1554028.638519113)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153049ac7990>\n",
      "Evaluating workflow:  96%|█████████▌| 2881/3000 [04:41<00:17,  6.81it/s]Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530499df4d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15310389e0b0>, 1554029.759071875)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530499dffd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304998fc90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530ff5c9b00>, 1554030.160450446)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530498a4190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530499a2ed0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153049be2290>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153049a9b790>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153049802d90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530ff6d1be0>, 1554028.577325637)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153049be2c90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15310389cad0>, 1554028.698747064)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153049802b90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530feabca60>, 1554028.881883804)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530499a3610>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530ff6d3460>, 1554028.773130155)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153049a9b2d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304977ffd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530ff5cb150>, 1554031.325069968)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304977fed0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530497b0b90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530ffbded60>, 1554031.646285433)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530497b04d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304988b790>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153049710c10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530542cad10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304972ca50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530ff9de510>, 1554030.2340088)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530498c4610>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530ff9df000>, 1554030.262960203)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153049710a10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530ff6d27b0>, 1554030.379077248)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304972d110>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530ff9dc2f0>, 1554030.468929066)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304988b450>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153049644750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530ffbdedd0>, 1554032.459947147)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153049644650>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153049500450>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303fbe1c50>, 1554033.978152973)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153049500390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530496d55d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303fa982f0>, 1554033.384438908)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530496d7390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304957cdd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056e05860>, 1554034.736910525)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15305428dcd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153049401350>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303fbe32a0>, 1554034.656380192)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153049400b90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153049463650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303fbe23c0>, 1554034.982277532)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304944aa10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153049362650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056ee51d0>, 1554036.999400094)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530495ccf10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304949c3d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303f323e70>, 1554034.639231039)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304949ca90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530492e6350>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15305744e120>, 1554039.438491949)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530492a9050>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304905a910>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530ffad8de0>, 1554040.331376789)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304905a7d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153048f1e150>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048f258d0>, 1554041.657605153)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153048f1e210>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153048e11010>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048e14f30>, 1554042.55213655)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153048e11250>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153048f85650>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304901fb10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048f26200>, 1554041.770225117)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153048f857d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048f26970>, 1554041.63139923)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153048f884d0>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153048e9ced0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048e16d60>, 1554044.78498269)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153048e9cb10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153048cda7d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048c0ff50>, 1554045.409309532)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153048cdaa10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153048de4410>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153048c5c2d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530ffadae40>, 1554044.934614464)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153048eda350>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048c0c670>, 1554044.916826167)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153048c5c950>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303f2864d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303f1eedd0>, 1554047.4834301)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303f279410>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303f2b9ad0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303f1efc40>, 1554047.471169904)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303f2ba050>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153048cab390>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048d14d00>, 1554047.478315338)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153048caaed0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303f0f9f10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303f11c650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303fbe2890>, 1554047.462503022)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303f2104d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048c0e200>, 1554047.459116164)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303f11c450>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153048c6c710>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303f1326d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303f2dd390>, 1554047.518405747)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153048c6c490>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303f2dd5c0>, 1554047.47560711)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303f132590>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303f145a50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048c0eb30>, 1554047.585217047)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303f145fd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303f0f8490>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303f149c90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303f2ddda0>, 1554047.447084004)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303f210410>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303f2de3c0>, 1554047.589755462)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303f14a310>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303f24b090>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303f195350>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303f2df000>, 1554048.415926601)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303f184150>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303f2df5b0>, 1554047.788139601)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303f197450>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303f1b9dd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303f2df850>, 1554047.828926381)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303f1b9990>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153048f79390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303f1d5a50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303f2dfaf0>, 1554047.797159341)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153048dd0bd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303f1dc0c0>, 1554048.382300961)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303f1d5850>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153049b65650>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153049911d90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153049940850>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153049a2d310>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153049b24710>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530499673d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153049abc690>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153049ad2950>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153049a5b3d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304998fd90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530499c6d90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153049980a10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304981e6d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153049852110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530498684d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304987dc90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153049873150>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304984c6d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530498dcd50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530498e5310>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304974cc90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530498f6190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304975dd50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153049775a90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304981e150>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530497bee90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530497961d0>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304987ee50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530498dd4d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304965d510>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530497f5250>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153049690650>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304974d6d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530496b43d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530496c8fd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530496bcc50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530495154d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153049539990>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304963d5d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153049684150>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530496eeb90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153049561850>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530495899d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153049517410>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530495a39d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530495cfb10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304959aed0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530495c7010>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153049450110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530495884d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530495ebb90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304942f450>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153049542710>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530494830d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530494df5d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530494b9d90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530494f7910>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153049318e50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153049328290>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304934c110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530493879d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530493a9390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530494bae50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153049369010>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153049385b50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530493c0f50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530493dc610>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153049554690>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153049211010>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153049222890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153049245210>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304925ba50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153049281350>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304924e590>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530492a8fd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304979fed0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530492e46d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153049542810>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530494baa90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304929f8d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530492e67d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153049111290>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153049112ed0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153049134ad0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304914cd50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153049188650>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153049283310>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530491a2050>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530491ae6d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530491c0c10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530491d5d90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530491f2c90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304901d4d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153049245d90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530490128d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153049038a50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530490755d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153049091fd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530490a9550>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530490b5c50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530490f31d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153048f08d10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530491f8090>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153048f34610>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530490e11d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153048f4b210>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153048f604d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530490ea090>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530493aa990>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153048fcaf90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153048fe2350>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153049092a90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530490fc210>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153048e1d4d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153048e36350>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153048e5d3d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153048e8fa10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153048f85810>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153048ec0e10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153048ed0250>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153048e7d490>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153048f62b10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153048f742d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153048d386d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153048f7b950>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153048fcbfd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153048d87c10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153048da1090>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153048dd0610>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153048dfdc90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153048d19f90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153048d61750>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153048de8b90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153048ddc8d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153048c2b210>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153048c67150>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153048c8bc50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153048caa250>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153048c8a510>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153048cca350>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153048cd8910>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303f228dd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153048cdf150>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153048cdc050>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303f20b4d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303f2473d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303f25a190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303f24ac50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303f2c9790>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fdcfd400>, 1554027.235513215)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153049b65710>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fdcfe120>, 1554028.161035397)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153049911c50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530feabf540>, 1554028.605922758)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153054248b50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530ff6d1010>, 1554028.083931234)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153049a2d050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530feabf3f0>, 1554028.854430853)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153049b24b50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530ff6d1fd0>, 1554028.251416701)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153049967290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fdae90f0>, 1554030.280210678)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153049abc490>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530ff5c8830>, 1554027.63598091)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153049ad2ed0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fe9b1010>, 1554029.216303314)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153049a5b450>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530ff6d15c0>, 1554028.55924465)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304998fc50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530ff6d06e0>, 1554029.078189591)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530499c6c50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530ff6d2d60>, 1554028.850233609)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304995d890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530feabd080>, 1554029.299909146)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304981e690>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15310389df60>, 1554029.623513509)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153049851f10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15310389e890>, 1554031.069475215)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153049869890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15310389d9b0>, 1554030.343532729)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304987dad0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15310389c980>, 1554030.154702645)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153049873290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530ff5cb310>, 1554029.468562376)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153049ae8590>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530ff9de120>, 1554030.473977289)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153049994d90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530ff6d3850>, 1554030.493765191)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530498e5890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530ff9df690>, 1554030.746246837)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304974d110>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530ff9dfe70>, 1554030.634800688)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153049be2bd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530ff9ddb00>, 1554031.064748233)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304975db50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530ff9dc520>, 1554031.086283969)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153049775fd0>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fe9b3000>, 1554031.082913961)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153049777dd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530ffbdd240>, 1554031.109591216)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530497bed50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530ff9de7b0>, 1554031.130108908)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153049795810>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530ffbdc280>, 1554031.974096919)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153049b56810>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530ffbddb70>, 1554032.192226257)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153049b2ed50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fe9b0b40>, 1554032.416595818)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304965d350>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303fbe0050>, 1554032.960469004)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530497f73d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303fbe2dd0>, 1554032.744513266)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153049690d90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303fbe3cb0>, 1554032.596806412)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304974dc50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303fbe39a0>, 1554033.149719248)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530496b3b50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303fbe3c40>, 1554032.959227354)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530496c8e90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303fbe3d20>, 1554033.540115055)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530498593d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303fbe2a50>, 1554033.024399463)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153049514e90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303fa99780>, 1554032.722541086)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153049539850>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530ffbdc440>, 1554032.592986622)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304963d3d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530ff6d3e70>, 1554032.308361294)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304973c890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303fa9bee0>, 1554032.752673462)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530496eea10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303fa98210>, 1554033.695423243)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153049561650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303fa9a6d0>, 1554033.60264284)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153049589850>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303fa99860>, 1554033.832505791)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530499ebe10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303fa98670>, 1554034.891161798)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530495a3890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303fa9a4a0>, 1554034.66085766)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530495cf9d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056e077e0>, 1554034.285415057)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304959afd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056e07380>, 1554034.392994751)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530495c6e10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056e05240>, 1554034.727142965)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304977fd90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056e07700>, 1554035.305289414)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153049aa4c90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056e078c0>, 1554034.533789384)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530495eba50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056e060b0>, 1554035.232097268)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304942f310>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056e07af0>, 1554035.32993885)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153049542610>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056e05470>, 1554035.502382606)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530496b1590>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303f3214e0>, 1554035.746864899)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530494df490>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303f321b00>, 1554036.035820963)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530494ba110>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303f320ec0>, 1554035.494573719)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153049502690>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303f320ad0>, 1554035.630727956)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153049318c50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303f3209f0>, 1554036.275345845)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530493285d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530feabd0f0>, 1554036.143481294)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304934c490>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056ee5d30>, 1554036.707993707)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153049387890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056ee7700>, 1554036.322555462)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530493a9ad0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303f322e40>, 1554035.367108954)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530494badd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530ff6d1630>, 1554036.480010967)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530493695d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303f320600>, 1554037.966829463)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153049385950>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056ee5c50>, 1554036.837015453)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530493c0d50>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056ee7d20>, 1554037.934290258)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530493dcd50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056ee48a0>, 1554037.316757727)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530493dea10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056ee6660>, 1554037.358877689)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530492114d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303fce6270>, 1554037.446693198)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153049222750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056ee4de0>, 1554037.958274593)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153049245890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303f323690>, 1554037.49217661)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304925bf10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056c52f20>, 1554037.69750594)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153049281210>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531079b8280>, 1554037.796457702)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304924e6d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15305734c910>, 1554037.804758805)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530492a9690>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530ff9ddbe0>, 1554038.475621125)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530492aab90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531079baa50>, 1554038.538919331)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530492e4dd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303fce6f20>, 1554037.268990443)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153049543890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056c534d0>, 1554037.41664447)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304942fb50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303fa980c0>, 1554038.056769587)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530492d9110>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15305734fe00>, 1554038.470118925)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530492e6310>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303f323ee0>, 1554039.090476427)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153049111090>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15305734d240>, 1554038.855213478)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153049362190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15305734cde0>, 1554038.660279292)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153049135890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15305734fa10>, 1554039.315801101)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530493c2750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15305744d780>, 1554039.044516253)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153049188d90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15310389c2f0>, 1554039.207433668)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153049282190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15305744cf30>, 1554039.367770532)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530491a1e50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15305734dcc0>, 1554039.286571615)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530491ac8d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303fbe2c80>, 1554039.304838989)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530491c0a10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15305744d7f0>, 1554040.22739553)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530491d5b90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15305744d390>, 1554039.532115104)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530491f2b50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15305744c8a0>, 1554040.115639387)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304901ebd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15305744c2f0>, 1554039.879385341)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153049245ed0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15305744dc50>, 1554039.810623942)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153049013190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15305744ec10>, 1554040.6264966)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153049038850>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530ffad9e80>, 1554040.544619548)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530490753d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530ffadb770>, 1554040.534822553)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153049092710>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15305734e270>, 1554040.839023213)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304909add0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530ffad9860>, 1554041.198988358)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530490b5a50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530ffadbe70>, 1554040.759267359)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530490f2fd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530ffad9390>, 1554041.50408926)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153048f08bd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048f24280>, 1554041.278605883)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530491f8310>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048f24590>, 1554041.357544056)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153048f34cd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048f24750>, 1554041.505184411)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530490e0490>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048f24de0>, 1554041.215748383)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153048f48d90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048f251d0>, 1554041.763597981)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153048f61ed0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303fa9af90>, 1554041.284830637)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530490ea910>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15305734f690>, 1554041.970790445)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530493aa550>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048f26f90>, 1554042.129345595)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153048fc8650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048f278c0>, 1554041.943118865)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153048fe26d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048f25a90>, 1554042.778235859)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153049092c10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530ffad9be0>, 1554042.391941655)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153048fdbf10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048e146e0>, 1554042.829276528)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153048e1e490>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048e14c90>, 1554042.573878954)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153048e36150>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530ff9dd320>, 1554042.974430762)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153048e5d290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048e14590>, 1554042.657539102)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153048f504d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048e169e0>, 1554043.141656214)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153048f85610>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303f323930>, 1554042.892003165)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153048ec0c10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048e171c0>, 1554043.222335767)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153048ed05d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048e15da0>, 1554043.49989408)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153048e8d4d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15305744d710>, 1554043.606659454)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153048f62f10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048d141a0>, 1554043.404026823)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153048d04710>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048d14d70>, 1554043.4694157)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153048ec1250>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048d15a90>, 1554043.580212406)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153048fe3510>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048d15da0>, 1554044.296429799)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153048fcbb50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048d16040>, 1554044.091230483)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153048d87ad0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048d16820>, 1554045.197954818)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153048da0ed0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303fa994e0>, 1554046.122399471)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153048d4f050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048e143d0>, 1554044.186178966)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153048dfdb50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048d14980>, 1554043.92466249)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153048d19e50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048d158d0>, 1554043.671147924)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153048d61590>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048d17e00>, 1554044.513699997)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153048de9590>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048e15160>, 1554044.659370022)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153048ddcb50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048d170e0>, 1554044.99289556)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153048e5e6d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048c0dda0>, 1554045.338356591)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153048c67010>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048c0e510>, 1554045.158550199)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153048c8bb10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048c0ee40>, 1554045.140531337)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153048caa050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048c0f1c0>, 1554045.187178873)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153048d59410>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048c0f3f0>, 1554045.993957512)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153048cca150>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048c0f9a0>, 1554045.754719204)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153048cd9010>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303f1ed320>, 1554045.813165528)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303f228c90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303f1ed630>, 1554047.187430384)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153048cdf550>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048c0d240>, 1554045.764513147)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153048cdc910>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048f25400>, 1554045.31505644)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303f20b350>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303f1ecec0>, 1554046.496255249)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303f247290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048d16ba0>, 1554047.174180481)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153048cdc890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048e17700>, 1554047.166283559)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303f24ba90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303f1efe70>, 1554047.179390716)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303f2c9610>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153049621a50>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153049556910>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530493cb850>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153049030990>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530ffbdcd00>, 1554032.046434439)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153049621910>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530ff5ca270>, 1554032.950718793)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153049556fd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056ee4a60>, 1554036.845904315)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530493cb710>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15305744d630>, 1554040.006839614)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153049030b90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303f0a9c50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303f1d8dd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303ef019b0>, 1554049.966983255)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303f0a9a90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303f2de200>, 1554050.240091094)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303f1d9f90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303f07aed0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303ef1f510>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303f1dde10>, 1554050.555466959)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303f07a610>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303f1dd320>, 1554049.968198671)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303ef1fcd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303f054fd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303f1dfbd0>, 1554049.957264018)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303f054590>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303f11db10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048c0f700>, 1554050.164246312)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303f11d010>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303f018310>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303f1ddcc0>, 1554050.829060141)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303f018e90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153048d187d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303f320fa0>, 1554051.385255182)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153048c6fd10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303ef93e90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153048e0d010>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303f1ef930>, 1554051.138964324)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303ef93110>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303f1eeac0>, 1554051.210179231)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153048e0ca50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303f001290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530ffbdfee0>, 1554051.471324144)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303f003dd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153048f889d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530493c8510>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fdbf97f0>, 1554051.00732715)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153048f74190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fdbfa5f0>, 1554051.247642417)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530490d1dd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530495e01d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530496b5e50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530ffbddef0>, 1554051.402869013)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153049563f50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530ff9dd400>, 1554051.545667285)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530496b5110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153049675510>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303f0ad590>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303f0b8e50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303f0b8990>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153048c6d4d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303ef38550>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303ef9ba90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303efaa490>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303efc1650>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303efc91d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303efd1190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303ef1ecd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303f17f250>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303ef84390>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048c0d4e0>, 1554049.572059032)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303efc1510>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303f1dc360>, 1554049.681554762)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303efc9090>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303f1dd630>, 1554051.600719017)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530496be6d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bf1d0f0>, 1554049.118359563)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303f085ad0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153103787a10>, 1554048.908380695)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303f0b9550>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048a4be00>, 1554049.101185059)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303f1cb310>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303f2dd780>, 1554049.399529072)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303ef32dd0>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530ff6d2a50>, 1554049.314784737)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303eefdd90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303f1ecf30>, 1554049.31087355)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303ef9b890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303f1efa80>, 1554049.403094738)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303efab0d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530ffadac10>, 1554049.800937724)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303efd1b10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303f2de890>, 1554049.062134601)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303eefd890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303f2dc980>, 1554049.424480291)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303f17fa10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303f2dda90>, 1554049.441542322)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303f0984d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530490787d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fdbf9710>, 1554051.329762713)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304960b810>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153049360c90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15305744fee0>, 1554052.589810382)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153049362a90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530499264d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303f1dfa80>, 1554052.068156638)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153049927e90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303f11cad0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153049a3ec90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057e91b70>, 1554051.946443127)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303f11db50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048d15940>, 1554051.937100695)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153049a3df90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153049207210>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153049a72a90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057e91a90>, 1554052.29308355)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303f279390>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd3467b0>, 1554051.820742993)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153049a73850>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304994dad0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd344f30>, 1554052.048669317)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304994c6d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153049a08c50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd223690>, 1554052.486260879)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153049a0a990>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303f24ad10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd223e00>, 1554052.75606399)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153049bfaad0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153049bda950>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303f2dfe00>, 1554052.812540499)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153049bdb650>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153049b7d1d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd220fa0>, 1554053.199436723)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153049b7e7d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530492fa650>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153049b03290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd2223c0>, 1554053.394316716)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530491c2c90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd220f30>, 1554053.795234414)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303fd69390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530541aa810>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153049832410>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303fbe3a10>, 1554053.085212185)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530541a8f10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fdbf8910>, 1554051.714951781)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153049832210>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153048c1f850>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304ba60fa0>, 1554053.843209068)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153049609ad0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303fd5c850>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15305412ba90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303f1df8c0>, 1554053.383750459)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153049a701d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304ba62350>, 1554053.694590951)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15305412bb10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530542b1e50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530ff9dd940>, 1554053.628137349)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530542b1950>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153054271710>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303fb45710>, 1554054.581081532)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153054273610>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304933cf90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530543dca10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303fb478c0>, 1554054.420206842)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304933de90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057e92580>, 1554053.586730305)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530543ddb90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303fd44ad0>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15305423b150>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15305431cbd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303fb47ee0>, 1554053.996855904)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153049b2dc90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303fb46e40>, 1554054.46359937)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15305431d010>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15305438cd50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd345630>, 1554054.159328701)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15305438c610>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153048f6d5d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303fb47770>, 1554054.131470998)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303f2526d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530541a1650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530ffbdc0c0>, 1554054.53500822)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530541a3f10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15305432d090>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304965a790>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530541a1f50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303fb47d90>, 1554054.315572831)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303fd45710>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531037c32a0>, 1554055.170360903)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15305432c710>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304ba60130>, 1554053.296021926)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153049b2d910>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303f1ef3f0>, 1554053.04048145)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530541a26d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153049b74410>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530543b5c50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fdbf9a20>, 1554055.151510923)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153054184050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531037c3ee0>, 1554054.828531203)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530543b6390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153054473fd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531037c29e0>, 1554055.736208092)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153054472f50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530544d4050>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530544b7750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531037c10f0>, 1554055.920488657)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530542cabd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531037c0280>, 1554055.536959095)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530544b7b50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153054463c10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303fb46dd0>, 1554055.854392881)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530544603d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153054460d50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b7c03d0>, 1554055.672448567)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153054142b50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153054451ed0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303fd3fa90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304ba62430>, 1554055.676947715)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153054452050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048c0eeb0>, 1554055.590468405)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303fd3f190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530545bda10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b7c0360>, 1554055.504734263)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530545be090>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530545bce90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b7c08a0>, 1554055.78956598)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530545bedd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303f054c50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530ffe17050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303fb45e10>, 1554056.92191447)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153049539d10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b7c04b0>, 1554056.478155914)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530ffe14390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530ffe3b790>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b659e10>, 1554056.369582556)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530ffe39b10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15305458d1d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530ffec2f50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530544ce010>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b7c3070>, 1554056.331057866)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15305458e550>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304ba62820>, 1554056.569954947)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153048ccafd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531037c02f0>, 1554054.769747333)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530544cd050>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530ffdc3250>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b65a190>, 1554056.642564275)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530ffdac0d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530544cfa10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b65b7e0>, 1554056.975046589)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530544cd790>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531003f05d0>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531009c66d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b658de0>, 1554057.605993561)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303f0da690>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b65a6d0>, 1554056.923159912)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531009c6190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530ffd00e90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531037c34d0>, 1554057.462023276)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530ffd003d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530ffc5ab90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b986e40>, 1554057.506179601)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530544726d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530543457d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153054408290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b984050>, 1554057.818870062)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153054463d10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b659860>, 1554057.322200479)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530ffc06e90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530ffc58f90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530ffa415d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b985710>, 1554057.495547917)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530ffc5a810>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b9877e0>, 1554057.440367008)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530ffa43ad0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530ffad2a50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b984360>, 1554057.817658894)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530ffad3cd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530ffa8c710>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530ffb08990>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531037c2f90>, 1554057.517783991)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530ffa8ddd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fe39ce50>, 1554057.935405709)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530ff9504d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530ff9c4e90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530ffe71a50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530ffd56990>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057e907c0>, 1554057.408841476)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530ff9c43d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b658a60>, 1554056.109703002)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530ffe73b50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b658ec0>, 1554056.368115641)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153054472dd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530ff5815d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303fb47930>, 1554058.581820309)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530ff578110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530ff969910>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530ffb940d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530ff5d6510>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fcee1ef0>, 1554057.958279623)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530ff96a310>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fcee0980>, 1554058.34492642)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530ffb95010>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530ff6c4a10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd13ce50>, 1554058.484086299)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530ff6c6cd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530ffe278d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd13dbe0>, 1554058.862142587)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fdba2c90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fea68110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fdbc3510>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd13e7b0>, 1554059.035848742)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fea6b2d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057e93c40>, 1554059.214029649)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fdbc1050>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530feb039d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057fdd710>, 1554058.735127822)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530feb00290>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530ffbceb90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd506dd0>, 1554059.153330343)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530ffbce9d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530ffa5cd90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fdca5f50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd507ee0>, 1554059.695929307)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530543fbf90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd507af0>, 1554059.591522751)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fdca40d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fdcb0b50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b986190>, 1554059.403788878)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fdcb17d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fda75350>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fdb0cd90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd0f1010>, 1554058.667862457)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530ffdb4f90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303f5cd160>, 1554059.071493315)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fdc4fc10>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303f5cf8c0>, 1554059.486941118)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fdb0fa50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fdc1d4d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fd9d85d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303f5ccc20>, 1554059.847167353)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fd9a2c10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303f5ccf30>, 1554059.828646669)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fd9dad10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fd97c9d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fd94c210>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fda467d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303f5ceeb0>, 1554060.027073412)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fd97e390>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303f5cde10>, 1554059.674880612)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fda704d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fd96f610>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303f5cd630>, 1554060.302779989)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fea5d4d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fd84f850>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fe437e70>, 1554060.409302641)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fd84cd50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153100373c10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fe435940>, 1554060.343959075)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fdba2650>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15310036ec10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530574e15d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd13fc40>, 1554061.061173822)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153057700990>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fe436890>, 1554060.704701322)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530574e2b10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153100a546d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153100bc1470>, 1554060.920492938)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153103047ad0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303facf810>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b9878c0>, 1554060.712221359)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303fa74410>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530573c6310>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153100a6e110>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153100bc08a0>, 1554060.934051709)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530573c6690>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b9854e0>, 1554061.203779937)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153100a6dbd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153042e2c310>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fcee0130>, 1554059.925070332)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fda46210>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fd809610>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd42e040>, 1554061.59858551)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153042e2e8d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303f5cdda0>, 1554059.472533177)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fd809a50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303fb9f310>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fc211dd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303f5cc910>, 1554061.216837399)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153103093210>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153100942590>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153100955f90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056ccc910>, 1554061.449483011)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fd9cc790>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057fdf380>, 1554061.307012995)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153100956b50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fcc0ff90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303fcd5e10>, 1554061.785404165)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531009f04d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15310093d650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531027511d0>, 1554061.655603535)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15310093e810>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303f2f47d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303f5144d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd6b6dd0>, 1554061.825932445)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153100462d10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b658440>, 1554061.559740801)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303f516f90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530573e2b10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153042e83e90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd6b60b0>, 1554061.546777393)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530573e09d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303f5ce900>, 1554061.705906812)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153042e83bd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153042f4ded0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303f5cf770>, 1554062.345122816)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153042f4c3d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303fbcd890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531051afbd0>, 1554062.147088574)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303fbcdf10>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153100a10a10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531051af770>, 1554062.560789449)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153100a10090>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fc8cf810>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531051af7e0>, 1554062.261390095)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fc8ceb10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531058ba3d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057fdc7c0>, 1554061.282277694)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303fb9e290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531051afe00>, 1554062.692566801)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fcf35bd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153057702c10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531051ad320>, 1554062.523619269)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fd14a910>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15305794b2d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303f5cf930>, 1554063.208575538)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303f300d10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530431e2e90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153105245710>, 1554063.369067499)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530431e3d90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303f539f50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304b77de50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153105244de0>, 1554063.258709454)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fd2a5b50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fccb1b00>, 1554063.778581945)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153057971250>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fcf2c3d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fccb3380>, 1554063.625866261)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fcf2de50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530428a3a10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153057be57d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd1d9cc0>, 1554063.247457482)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530544b7d50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd6b49f0>, 1554063.115627006)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153106645890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530578a5610>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531034df810>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fc7c8830>, 1554063.878206091)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531003fd290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fc7c9a90>, 1554063.744399727)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531034dc250>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304bad2690>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd507380>, 1554063.618450484)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304bad1ad0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304b5ba190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531034c2250>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fc7cacf0>, 1554063.881877304)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fc136710>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fc7ca890>, 1554063.78760737)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304b5ae6d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153100a01490>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd75b700>, 1554063.717936825)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153100a00350>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303fb7bc50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fcd08f50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd75ae40>, 1554064.09973741)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fc9579d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531051aff50>, 1554062.427653441)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153102136a90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153103948f50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd759cc0>, 1554064.587076378)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153103948d10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153056c7e6d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153103766610>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531051ae430>, 1554064.1945652)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153056cd6d10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd6b69e0>, 1554064.049520546)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153103765f90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531036f2c90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bc28600>, 1554064.772974835)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153056d5e750>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531036f0390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531036df9d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bc284b0>, 1554064.228649512)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15310093f850>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bc28210>, 1554064.438792429)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531036dce10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fd831a50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fc5239a0>, 1554065.079095902)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fd830690>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531038efb10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531038d1fd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bc2a970>, 1554065.360250627)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531038d0050>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153103999c50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531052454e0>, 1554064.995755554)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531038a2a10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153103d604d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b856350>, 1554065.319800337)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530428b2750>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153048911990>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bc2a900>, 1554065.436167699)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153104faddd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153048abe410>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153104f58950>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bc299b0>, 1554065.465459101)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fda20190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531044cf110>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b6583d0>, 1554065.697966138)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15310395cbd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531051ad6a0>, 1554065.617112041)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153104f58450>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057fde120>, 1554065.819573985)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153104ebb690>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15310453f890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b857cb0>, 1554065.812133151)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15310453c190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fc224610>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153103993fd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056cccec0>, 1554066.017626494)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fc226790>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b855860>, 1554066.634030928)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15310449ead0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15310596e8d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153104ed7850>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd6b6740>, 1554065.936684691)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15310093e590>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fca3cfa0>, 1554065.958428614)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153104ed5510>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153104e9a190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530ff6d2660>, 1554066.118665745)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304bf6ce90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153048a81810>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304b329890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bc29400>, 1554066.09998626)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153048a83c10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153056694490>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd5db930>, 1554066.718754326)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530566960d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304b306750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048e15a20>, 1554066.560419238)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304b307a10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153048a29990>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fca3e970>, 1554067.464480981)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153048a296d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304bffe110>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bc2a660>, 1554066.896402898)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304bffeed0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304b34f690>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fda12510>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bcab4d0>, 1554066.90715435)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fd5c5110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304b474310>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fc07bc90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bc29f60>, 1554066.674830628)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304bf16c10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bcaa6d0>, 1554067.519024778)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fd9a7550>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bca9240>, 1554067.045048131)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304b4767d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530ff6d1240>, 1554065.807487276)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153056d7cad0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304b535a90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bca8980>, 1554067.557584507)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304b5f7e90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fdd08dd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bcaacf0>, 1554067.727313886)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fdd09910>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530feb12490>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530ff5ca350>, 1554067.655965213)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530feb11790>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530ff724250>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530ff5c9710>, 1554067.772618674)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530ffa7e610>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530ff725d90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153057359150>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530ff5c9b70>, 1554070.992441606)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530ff724a10>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530ff5ca970>, 1554068.288378721)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15305735b610>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fdce6cd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153054df9490>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530feabec10>, 1554067.485688714)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fdce6210>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530feabee40>, 1554067.996736939)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530ffd254d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153057258910>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd42e9e0>, 1554067.867408374)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153104ed54d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15310395fb90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530561bf090>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530feabff50>, 1554068.08533014)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531038ef390>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530feabed60>, 1554068.074096038)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530561bd150>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fdbe1950>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fdc2b0d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531051ad470>, 1554068.272277674)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fdbe0990>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bc292b0>, 1554068.724867814)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fdc28510>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fe9d2dd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fe9b18d0>, 1554068.398129107)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fe9d1410>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153054c35950>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15310742b950>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fe9b3700>, 1554068.545843989)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153054c359d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fe9b1e10>, 1554068.658010363)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531043f1f10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fdc05210>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530574d98d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fe9b0c90>, 1554068.697468742)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fdc04350>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530561b6f90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fe9b0130>, 1554069.23951613)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530561b4750>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530ff5cfb50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153054c50dd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bca9550>, 1554069.243490755)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530ff5cf610>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153054dd4650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fdcff310>, 1554069.723805008)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153054dd7f10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153054eae690>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bcab070>, 1554069.574760607)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153054eaf090>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153054c24f90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fdcfe9e0>, 1554069.263494637)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153054c25510>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153054c2fb90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fdcfdd30>, 1554069.425827723)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153056697310>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153054ebee10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153054ecb550>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fdcfdef0>, 1554069.648695347)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153054ebf3d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fdcff1c0>, 1554070.281671745)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153054ecaa90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fc2dff50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fdaeae40>, 1554070.001369296)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fc2dcad0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530561bd4d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fdae8ad0>, 1554069.926111283)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304b287450>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15310447dd90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fd8c0850>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fe9b14e0>, 1554069.290641667)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530574da110>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bca86e0>, 1554069.363576561)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153054c50990>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fdaea3c0>, 1554070.515096531)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15310447f590>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fdaebd20>, 1554070.420985036)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fd8c21d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153054c19b90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fdaebe00>, 1554070.56675714)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153054c1b290>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303fcf94d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153054d74890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fdaeb380>, 1554070.354988521)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fd9845d0>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b854130>, 1554070.925216672)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153103bfd610>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530574be850>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fe9b20b0>, 1554070.387058761)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530574bc150>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530561230d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15310398ab90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530551860b0>, 1554071.242429031)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304b363350>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153055186820>, 1554071.218220238)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15310398ae50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153054c74c10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153054ff72d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153055187af0>, 1554071.50199704)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153054c771d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153055185a90>, 1554070.9444711)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153054ff5190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153054fb1bd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153055186cf0>, 1554071.065258295)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153054fb1950>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153056088e10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530ff5cb2a0>, 1554071.210558714)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304bfca4d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153054d22610>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153054f68ed0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fdcfc210>, 1554071.432500201)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153054d8add0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153055185d30>, 1554072.039039512)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153054f68e50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304ba65510>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bba54e0>, 1554071.848304565)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304b3f2090>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153054f10cd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530550e6610>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153054d822d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fdcfc6e0>, 1554071.955285429)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153054f109d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bba50f0>, 1554071.911792542)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153054e74750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fdae80c0>, 1554070.320679382)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153054d83790>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153054e0c4d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bba41a0>, 1554072.584338532)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153054e0c890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153054d80ad0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153055000990>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fdaea4a0>, 1554072.416938585)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531062f2310>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bba7cb0>, 1554072.546087462)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530550022d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530550bad10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153055028d50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fdaeb230>, 1554072.511038607)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530550d9710>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bba5d30>, 1554072.881687392)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15305502bad0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530551db8d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530551b3010>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bba67b0>, 1554072.811863826)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530551db650>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530550dff90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bd8fcb0>, 1554072.812922638)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530550dfd50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15305518a290>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530550cc290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bd8ec10>, 1554072.380525967)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15305518b790>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bd8dc50>, 1554072.662885339)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530550cc8d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530551335d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bd8e0b0>, 1554073.079347456)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153055133590>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530552f34d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bd8f690>, 1554072.951950379)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530552f3310>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15305525fb50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530552d1610>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bd8d630>, 1554073.106024654)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15305525d610>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bcaa040>, 1554074.14983868)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530552d0090>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530552f4490>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530551eeed0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bca8ec0>, 1554072.369433733)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530551b1710>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bd8f230>, 1554073.321060919)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530428fd890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153055184590>, 1554073.170965927)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530551ef210>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153055281650>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530553f0ed0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153055184910>, 1554073.908750992)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530552801d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153055093000>, 1554074.087967748)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530553f2c90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153055247290>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530553b6d90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153055092120>, 1554073.881173828)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153054f0add0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b857f50>, 1554073.978580903)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530553b7550>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530553442d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530550912b0>, 1554074.648951991)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153055344450>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530553a1490>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530554f3550>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fe9b2350>, 1554074.834989691)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15305530ee90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153055091a20>, 1554073.955831972)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153055133950>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530554a5810>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15305514c110>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fdaeb000>, 1554074.952462172)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15305514f290>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153054d80b50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153055466990>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd9d6270>, 1554074.563784015)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153054d81890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530551e4410>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153055369c90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd9d65f0>, 1554074.773421546)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15305540add0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd9d71c0>, 1554076.870819153)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153055369bd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15305532c890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153055091390>, 1554074.922089774)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530554a5890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd9d7d20>, 1554074.495367745)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153054c24d50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd9d7b60>, 1554074.796210213)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153055406810>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153055464750>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530555a6b90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd9d5320>, 1554074.715440853)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153055464810>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd9d7380>, 1554075.314961023)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530555a6510>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153055442fd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15305553f790>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd9d5080>, 1554075.334538415)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153055442110>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd9d7310>, 1554075.115946354)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153055572190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530556ac1d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd7d1860>, 1554075.90972775)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530556ae610>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153054c19090>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530556ba310>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd7d0980>, 1554075.767972404)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15305550bc50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd7d1630>, 1554076.409313607)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530556b9710>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153055522e90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530555eba50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd7d2890>, 1554076.051214698)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530556ac6d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153055187070>, 1554075.799340997)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530555e8f90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530552c9650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd7d38c0>, 1554076.179779853)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530552c95d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530556146d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd7d0f30>, 1554076.430619815)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530557e4610>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530557c02d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530557a9a10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303f52baf0>, 1554076.661974681)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530557a9ad0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153055774d90>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530555b04d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530550907c0>, 1554076.312767055)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530557c1a90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303f5286e0>, 1554076.854177172)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153055775d90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd9d6580>, 1554074.896348173)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530555b0550>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153055320b90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530557491d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303f52b3f0>, 1554077.012414818)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153054d77150>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15305572f910>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303f52a3c0>, 1554076.922359237)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15305572d290>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153055791610>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530558e7c50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303f52af20>, 1554076.571631374)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153054db0890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd7d0b40>, 1554076.825508881)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530558e6350>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153055608bd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd9d78c0>, 1554076.637311757)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153055609a50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153055880190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153055841b50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd9d7700>, 1554077.002631448)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153055571390>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531051533f0>, 1554077.0452419)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153055843850>\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153055847050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153105153b60>, 1554077.146597881)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530558443d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530559d3a10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bd8dbe0>, 1554077.960676665)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530559d2c50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530559b4210>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531051501a0>, 1554077.768252582)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530559b7190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153055995890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153055977e10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153105150210>, 1554078.102062442)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153055669c90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd7d23c0>, 1554078.568529254)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153055977050>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153055918a10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153105153700>, 1554078.213435156)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153055919cd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153055617d50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153105152430>, 1554077.883653736)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153055985190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153055907a10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153055acbdd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bd8f460>, 1554076.742430633)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153055749b50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd7d3af0>, 1554078.196416215)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530551b1410>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bb22cf0>, 1554078.344477386)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153055aca210>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153055aace10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153055a6e190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303f52bc40>, 1554078.652326653)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153055aacfd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153055a1db50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153105153310>, 1554078.866873758)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153055a1ff50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153056140110>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bb22200>, 1554078.882439207)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153056142350>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153055acb050>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153104fe2510>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bd132a0>, 1554078.499308541)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153055a12b50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bd10b40>, 1554078.87988634)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153104fe03d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530558eaf50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153055b93ad0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303f52a6d0>, 1554078.859336807)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530558e9cd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153055b80150>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bba4130>, 1554078.875590581)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153055b81c10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153055843950>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153055ad6510>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bd10670>, 1554079.003573665)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530554ca310>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bd106e0>, 1554079.264660078)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153055ae41d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530559d0390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153055cd0110>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153105151cc0>, 1554079.46856931)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530559d3850>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd7d3150>, 1554079.998185433)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153055cd0fd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153055cacb90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bb20de0>, 1554078.552754583)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153055a5eb90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153055a56490>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bd11080>, 1554078.592646928)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153055b90a10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303f4741a0>, 1554079.889876385)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153055cac350>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bb23690>, 1554078.46037698)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153055a55410>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153055cacc90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303f4744b0>, 1554080.555308722)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153055ca5a10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15305514d650>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153055c635d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303f475cc0>, 1554079.93309277)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153055c7f110>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303f52acf0>, 1554080.626413123)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15305573aed0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530560583d0>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bd12890>, 1554080.295470618)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153056173550>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153055c271d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153055c0f690>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303f529ef0>, 1554080.735905727)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153055c262d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303f477230>, 1554080.88287238)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153055c0ef50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153055f7c5d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153055d066d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303f529e10>, 1554080.140925308)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153055f7c450>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303f474910>, 1554080.569853335)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153055d06990>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153055e70290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531050f9710>, 1554080.323127764)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153055e707d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153102fcdad0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531050fa660>, 1554080.613572313)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153102fce190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153055fdb350>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153055e10250>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531050f8d00>, 1554081.194109069)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15305616add0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531050fac80>, 1554081.18231719)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153055e13fd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153057236610>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153055d8b750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531050fa5f0>, 1554080.351286907)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153055385b50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531050fb000>, 1554080.703230424)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153055daa590>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153055d84590>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531050fbd90>, 1554080.747733458)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153055d87750>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153055d5d250>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531050fae40>, 1554081.117382695)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15305582ec10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530556ae710>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531050fbee0>, 1554081.126376637)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530559d1290>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153055f99c10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153055986f10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd8d4a60>, 1554082.521661148)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153055aba850>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd8d40c0>, 1554081.673554509)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153055c9f010>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153055eeb2d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd8d65f0>, 1554082.384077057)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153055eeaa50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153055ce84d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd8d54e0>, 1554082.696882342)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153055d4e2d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153055f6d890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153055dda790>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd8d6d60>, 1554082.107870204)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303ef2b410>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd8d5010>, 1554082.788989277)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153055c604d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530542baed0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd8d4980>, 1554082.605251487)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530542bb910>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303efd0890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd8d6f90>, 1554082.656595986)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303efd0ed0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303e7590d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303e7754d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd8d6b30>, 1554082.65116831)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303e758fd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531051511d0>, 1554082.587271493)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303e774250>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303ef2b650>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303e7947d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057f422e0>, 1554082.405344151)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153055b02190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303e79ce50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153055eda1d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531050fa9e0>, 1554080.785704653)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153055edabd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bb204b0>, 1554082.45737445)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153055a54c90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303f52bee0>, 1554082.401352771)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303e7966d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153055d84610>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303e7bb8d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057f41a20>, 1554082.794070723)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153055d84910>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b2624a0>, 1554082.975609485)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303e7bb790>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530572344d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531050f91d0>, 1554083.082805912)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303ef36b90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303e5ecc10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b260ad0>, 1554082.933692257)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153055e11850>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303e5eed10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b260c90>, 1554082.759735724)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303e5eebd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15305595be50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303e632010>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b260910>, 1554082.873321955)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303e7d0150>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bb239a0>, 1554083.356796116)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303e631ed0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303e79cd90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b260c20>, 1554083.822909858)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303e79d1d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303e619850>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153055b2d910>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd8d6c10>, 1554083.928032887)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153055677510>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b358e50>, 1554084.793335552)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303e63f490>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303e6421d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b3599b0>, 1554084.179690614)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303e642e90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303e6b0850>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b35add0>, 1554084.289201579)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303e6b0ad0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303ef41b10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303e6de9d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303e6068d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b261be0>, 1554082.763267119)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303e6066d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bd12900>, 1554084.35109926)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303ef43910>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b35a900>, 1554084.341202028)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303e6de7d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303e5ed810>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056d69390>, 1554084.586173415)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303e508b50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303e534c90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056d6a740>, 1554084.842559279)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303e534b50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303e54ee90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056d68210>, 1554084.624207608)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303e68f3d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303ef2a150>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303e569b10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303f4755c0>, 1554084.693487133)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303e554b50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057f40830>, 1554084.869192198)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303e56a250>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303e57aa10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153103785400>, 1554084.978854727)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303e57a810>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303e6ce510>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303e5a1050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b358c90>, 1554084.849559893)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303e6ce110>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153103785470>, 1554084.819750775)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303e5a0f10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303e51ba90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303e5c6610>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153103785fd0>, 1554085.541812317)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303e51b910>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531037840c0>, 1554085.576430369)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303e5c6410>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303e5daf90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303ef52e90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b262040>, 1554085.512789261)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303ef519d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303e54ce10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303e76cf90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303e519850>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303e536190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd8d6270>, 1554085.848672227)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303e5dae10>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056d6a510>, 1554084.319418995)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303e519650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056d6ba80>, 1554084.415649001)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303e670d50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b2602f0>, 1554087.813449105)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303e54e090>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b35aeb0>, 1554084.633587913)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153055d51790>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303e434d50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303e44b210>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303f4774d0>, 1554085.936026015)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303e437650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b45c2f0>, 1554086.2505881)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303e44b0d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303e458850>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b45c3d0>, 1554086.056988478)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303e6b0b50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303e45a3d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b359ef0>, 1554086.478565866)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303e45a650>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303e47d290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b45e040>, 1554086.772180303)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303e47d150>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303e498510>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303e4bac90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153103787a80>, 1554086.214748383)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303e4982d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b45d780>, 1554086.645031557)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303e4bafd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303e4c9990>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b45cd00>, 1554086.289360108)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303e4c9850>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303e420890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303e2ed110>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b45f930>, 1554086.591094974)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303e423f90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303e2eff90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b55a040>, 1554086.939096181)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303e2efe50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303e2fba50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303e311d50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b55a820>, 1554086.853263367)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303e2fa450>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b55add0>, 1554086.783432779)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303e311b50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303e48fb10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303e332610>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b558600>, 1554086.748264882)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303e48fb50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b5596a0>, 1554086.650230074)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303e331210>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303e5c6690>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b55a2e0>, 1554087.051463992)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303e36d250>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056d69e80>, 1554086.580280054)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303e2ed890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b55a350>, 1554087.742780248)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303e5c65d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303e459710>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b5594e0>, 1554087.569635888)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153055846e90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303e39ed10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531037865f0>, 1554087.701506081)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303e39eb90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303e3889d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303e3cc990>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b35bcb0>, 1554087.718869281)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303e37a190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b55a6d0>, 1554088.255838697)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303e3cc790>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303e39ded0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303e3e6b50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b55b0e0>, 1554088.064177052)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303e39d0d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056678130>, 1554088.454758286)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303e3e6a10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303e224310>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b45d7f0>, 1554088.358188484)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303e204890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303e22e450>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15305667bbd0>, 1554088.403908103)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303e22e310>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530554b1d50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303e251ed0>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056679d30>, 1554089.235726114)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303e23c190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048a488a0>, 1554088.913127489)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303e251d90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303e226a50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b45dd30>, 1554088.467082827)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303e2119d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303e36e510>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303e291910>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd8d5d30>, 1554088.939130779)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303e3b6650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048a49550>, 1554088.527623408)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303e2917d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303e4a2dd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303e376150>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048a49fd0>, 1554088.33035885)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303e4a2cd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b5588a0>, 1554087.126326886)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303e375f50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303e23f910>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048a49780>, 1554088.583663732)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303e23ef90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303e26e810>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303e2df290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048a49e10>, 1554088.974301065)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303e3b6cd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048a4b460>, 1554089.348803532)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303e2df150>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303e359250>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048a4b700>, 1554089.717180171)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303e35a550>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303e109a90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303e129a10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530488b90f0>, 1554090.731663517)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303e109510>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b559e10>, 1554089.434643029)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303e129910>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303e100c90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303e14d390>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531037852b0>, 1554090.085743229)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303e100290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530488b9ef0>, 1554089.662665223)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303e14d250>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303e16d790>\n",
      "Evaluating workflow:  96%|█████████▌| 2883/3000 [04:41<00:16,  7.06it/s]Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b45e5f0>, 1554090.069680714)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303e16e250>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303e19ddd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531050f8f30>, 1554090.276647016)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303e19e510>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303e22f290>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303e1b9550>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15305667ac10>, 1554089.967196823)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303e22e510>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303e1c2f50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530488ba200>, 1554090.371175243)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303e1c2e10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303e17ea90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303e1e1610>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303e2c4190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530488ba120>, 1554090.230705178)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303e1b9350>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048975be0>, 1554090.888261329)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303e17ebd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048976f20>, 1554090.078436518)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303e1e1cd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048a4a660>, 1554088.462575358)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303e2c4510>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303e266d10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048974130>, 1554090.306910528)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303e266d50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303e000610>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303e225610>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048974210>, 1554090.143064218)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303dff7f50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530489778c0>, 1554090.07718458)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303e224810>\n",
      "Evaluating workflow:  96%|█████████▋| 2890/3000 [04:41<00:07, 14.60it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  96%|█████████▋| 2893/3000 [04:42<00:09, 11.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  97%|█████████▋| 2896/3000 [04:42<00:08, 11.96it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  97%|█████████▋| 2898/3000 [04:42<00:07, 12.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  97%|█████████▋| 2900/3000 [04:43<00:11,  8.37it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  97%|█████████▋| 2904/3000 [04:43<00:10,  9.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  97%|█████████▋| 2909/3000 [04:43<00:08, 10.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  97%|█████████▋| 2911/3000 [04:43<00:07, 11.55it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  97%|█████████▋| 2915/3000 [04:44<00:06, 12.87it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  97%|█████████▋| 2917/3000 [04:44<00:07, 11.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  97%|█████████▋| 2921/3000 [04:44<00:07, 10.04it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  97%|█████████▋| 2923/3000 [04:45<00:09,  7.96it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  98%|█████████▊| 2927/3000 [04:45<00:07, 10.35it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  98%|█████████▊| 2929/3000 [04:45<00:07,  9.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  98%|█████████▊| 2936/3000 [04:46<00:03, 16.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  98%|█████████▊| 2939/3000 [04:46<00:05, 11.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  98%|█████████▊| 2941/3000 [04:46<00:06,  8.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  98%|█████████▊| 2943/3000 [04:47<00:05,  9.55it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  98%|█████████▊| 2947/3000 [04:47<00:06,  8.61it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  98%|█████████▊| 2954/3000 [04:47<00:03, 15.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  99%|█████████▊| 2957/3000 [04:48<00:02, 16.14it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  99%|█████████▊| 2960/3000 [04:48<00:04,  9.49it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  99%|█████████▉| 2964/3000 [04:49<00:03,  9.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  99%|█████████▉| 2966/3000 [04:49<00:04,  7.74it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  99%|█████████▉| 2968/3000 [04:49<00:03,  9.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  99%|█████████▉| 2972/3000 [04:50<00:02, 10.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  99%|█████████▉| 2974/3000 [04:50<00:02, 12.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  99%|█████████▉| 2979/3000 [04:50<00:01, 12.80it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  99%|█████████▉| 2981/3000 [04:50<00:01, 13.71it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  99%|█████████▉| 2983/3000 [04:50<00:01, 11.63it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow: 100%|█████████▉| 2985/3000 [04:51<00:01,  8.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow: 100%|█████████▉| 2988/3000 [04:51<00:01,  8.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow: 100%|█████████▉| 2992/3000 [04:51<00:00, 10.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow: 100%|█████████▉| 2998/3000 [04:52<00:00, 13.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow: 100%|██████████| 3000/3000 [04:52<00:00, 10.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "\u001b[32m2026-01-04 10:32:41.682\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36m__main__\u001b[0m:\u001b[36m<module>\u001b[0m:\u001b[36m4\u001b[0m - \u001b[1mEvaluation metrics (before optimization): {'f1': 0.0, 'em': 0.0, 'acc': 0.413}\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "logger.info(\"Evaluating workflow on test set...\")\n",
    "with suppress_logger_info():\n",
    "    results = textgrad_optimizer.evaluate(dataset=benchmark, eval_mode=\"test\")\n",
    "logger.info(f\"Evaluation metrics (before optimization): {results}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "0f33f493",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'f1': 0.0, 'em': 0.0, 'acc': 0.413}"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "ceebbf97",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "46206f6d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "500"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(textgrad_optimizer.evaluator._evaluation_records)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "713d6040",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   0%|          | 1/500 [00:01<14:52,  1.79s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   0%|          | 2/500 [00:02<07:50,  1.06it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   1%|          | 4/500 [00:02<03:34,  2.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.3333333333333333, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   1%|▏         | 7/500 [00:02<01:30,  5.46it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   2%|▏         | 11/500 [00:03<01:02,  7.84it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   3%|▎         | 13/500 [00:03<00:51,  9.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.75, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   3%|▎         | 15/500 [00:03<01:04,  7.47it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   4%|▎         | 18/500 [00:04<01:26,  5.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.4, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.18181818181818182, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   4%|▍         | 20/500 [00:04<01:07,  7.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.7272727272727273, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   5%|▌         | 25/500 [00:05<01:27,  5.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   5%|▌         | 26/500 [00:05<01:22,  5.71it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.2857142857142857, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   6%|▌         | 29/500 [00:06<01:12,  6.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   6%|▌         | 31/500 [00:06<01:19,  5.93it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   6%|▋         | 32/500 [00:06<01:32,  5.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.4444444444444445, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   7%|▋         | 35/500 [00:07<01:13,  6.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   7%|▋         | 37/500 [00:07<01:11,  6.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   8%|▊         | 38/500 [00:07<01:42,  4.52it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   8%|▊         | 40/500 [00:08<01:46,  4.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.5, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.13333333333333333, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   9%|▉         | 45/500 [00:08<00:50,  8.97it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.8, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   9%|▉         | 47/500 [00:09<00:59,  7.55it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  10%|▉         | 49/500 [00:09<01:23,  5.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  11%|█         | 55/500 [00:10<01:19,  5.61it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  11%|█         | 56/500 [00:10<01:15,  5.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  12%|█▏        | 59/500 [00:11<01:05,  6.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.125, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  13%|█▎        | 63/500 [00:11<00:47,  9.17it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.33333333333333337, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  13%|█▎        | 65/500 [00:12<01:07,  6.42it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  13%|█▎        | 66/500 [00:12<01:18,  5.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  14%|█▎        | 68/500 [00:12<01:11,  6.05it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  14%|█▍        | 69/500 [00:12<01:20,  5.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  14%|█▍        | 72/500 [00:13<01:11,  6.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.8, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  15%|█▍        | 74/500 [00:13<01:03,  6.72it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  15%|█▌        | 75/500 [00:13<01:02,  6.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  16%|█▌        | 78/500 [00:14<01:09,  6.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  16%|█▌        | 80/500 [00:14<01:07,  6.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.3333333333333333, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  17%|█▋        | 85/500 [00:15<00:45,  9.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.30769230769230765, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.23529411764705882, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.10526315789473684, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  17%|█▋        | 87/500 [00:15<00:54,  7.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.16666666666666669, 'em': 0.0, 'acc': 0.0}\n",
      "\u001b[32m2025-12-09 13:03:55.422\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'_id': '5ac1276c554299294b2190bc', 'answer': 'Euthanasia Program', 'question': \"Heinrich Gross participated in the Nazi regime's Aktion T4 more commonly referred to as what?\", 'supporting_facts': [['Heinrich Gross', 0], ['Aktion T4', 3]], 'context': [['Karl Freiherr Michel von Tüßling', ['Karl Freiherr Michel von Tüßling (27 July 1907 – 30 October 1991) was a Schutzstaffel (SS) officer who served in the Nazi government of German dictator Adolf Hitler and in the SS Main Office.', ' From 1936 onwards, he was the personal adjutant of \"Reichsleiter\" and SS-\"Obergruppenführer\" Philipp Bouhler, who was in charge of Hitler\\'s Chancellery (Kanzlei des Führers), head of the euthanasia programme Aktion T4, as well as co-initiator of Aktion 14f13.', ' In 1947 Tüßling provided an affidavit in defence of war criminal Viktor Brack who was sentenced to death at the Nuremberg trials.']], ['Heinrich Gross', ['Heinrich Gross (14 November 1915 – 15 December 2005) was an Austrian psychiatrist, medical doctor and neurologist, a reputed expert as a leading court-appointed psychiatrist, ill-famed for his proven involvement in the killing of at least nine children with physical, mental and/or emotional/behavioral characteristics considered \"unclean\" by the Nazi regime, under its Euthanasia Program.', ' His role in hundreds of other cases of infanticide is unclear.', \" Gross was head of the Spiegelgrund children's psychiatric clinic for two years during World War II.\"]], ['Am Spiegelgrund clinic', [\"Am Spiegelgrund was the name of a children's clinic in Vienna where 789 children were killed under the Nazi Regime Children's Euthanasia Program, also known as Aktion T4.\", ' Between 1940-1945, the clinic operated as part of the psychological institution “Am Steinhof” (renamed the Otto Wagner Clinic) on the Baumgartner Höhe, now located in Penzing, the 14th district of Vienna.', ' This clinic was divided into a reform school and a sanatorium for children, which included a so-called Children’s Ward, where sick, disabled, and otherwise ‘un-educable’ adolescents were abused and subjected to harsh medical experiments.', \" Some died by lethal injection and gas poisoning; others by disease, undernourishment, exposure to the elements, and 'accidents' relating to their conditions.\", ' The brains of up to 800 victims were preserved in jars and housed in the hospital for decades.']], ['Philipp Bouhler', ['Philipp Bouhler (11 September 1899 – 19 May 1945) was a senior Nazi Party official who was both a \"Reichsleiter\" (National Leader) and Chief of the Chancellery of the Führer of the NSDAP.', ' He was also an SS-\"Obergruppenführer\" in the \"Allgemeine SS\" who was responsible for the Nazi \"Aktion T4\" euthanasia program that killed more than 70,000 handicapped adults and children in Nazi Germany, as well as co-initiator of \"Aktion 14f13\", also called \"Sonderbehandlung\" (\"special treatment\"), that killed 15,000–20,000 concentration camp prisoners.']], ['Artur Hojan', ['Artur Hojan (7 August 1973 – found dead, 9 February 2014) was a journalist and published author specializing in the history of the Chełmno extermination camp and the Nazi involuntary euthanasia programme conducted in the territory of occupied Poland by the SS during World War\\xa0II.', ' Hojan was the co-founder of the \"Tiergartenstrasse4\" Association in 2005 (together with Cameron Munro) devoted to Aktion T4 history, with emphasis on the Kościan psychiatric hospital located where he lived.', ' Hojan, age of 40, left home in the evening of 1\\xa0December 2013 at 8\\xa0p.m. for a walk around town and disappeared.', ' His body was found two months later on 9\\xa0February 2014 floating in the Obra canal near the town of Kiełczewo, and identified later.', ' The cause of death has not been determined.', ' He was buried at the Kościan cemetery on 15\\xa0February 2014.', ' He left behind a wife and young daughter.', ' The monograph \"Treblinka Death Camp: History, Biographies, Remembrance\" by Chris Webb, the co-founder of H.E.A.R.T (also known as the HolocaustResearchProject.org), is dedicated to his memory.']], ['Aktion T4', ['Aktion T4 (German, ] ) was a postwar name for mass murder through involuntary euthanasia in Nazi Germany.', ' The name T4 is an abbreviation of \"Tiergartenstraße 4\", a street address of the Chancellery department set up in the spring of 1940, in the Berlin borough of Tiergarten, which recruited and paid personnel associated with T4.', ' Certain German physicians were authorized to select patients \"deemed incurably sick, after most critical medical examination\" and then administer to them a \"mercy death\" (\"Gnadentod\") .', ' In October 1939 Adolf Hitler signed a \"euthanasia decree\" backdated to 1 September 1939 that authorized his personal physician Karl Brandt and \"Reichsleiter\" Philipp Bouhler to implement the programme.']], ['Karl Brandt', ['Karl Brandt (January 8, 1904 – June 2, 1948) was a German physician and \"Schutzstaffel\" (SS) officer in Nazi Germany.', \" Trained in surgery, Brandt joined the Nazi Party in 1932 and became Adolf Hitler's escort physician in August 1934.\", ' A member of Hitler\\'s inner circle at the Berghof, he was selected by Philipp Bouhler, the head of Hitler\\'s Chancellery, to administer the \"Aktion T4\" euthanasia program.', ' Brandt was later appointed the Reich Commissioner of Sanitation and Health (\"Bevollmächtigter für das Sanitäts- und Gesundheitswesen\").', ' Accused of involvement in human experimentation and other war crimes, Brandt was indicted in late 1946 and faced trial before a U.S. military tribunal along with 22 others in \"United States of America v. Karl Brandt, et al\".', ' He was convicted, sentenced to death, and later hanged on June 2, 1948.']], ['Gerhard Kretschmar', ['Gerhard Herbert Kretschmar (20 February 1939 – 25 July 1939), was a German child born with severe disabilities.', \" After receiving a petition from the child's parents, the German Führer Adolf Hitler authorized one of his personal physicians, Karl Brandt, to have the child killed.\", ' This marked the beginning of the program in Nazi Germany known as a \"euthanasia program\" (Aktion T4) which ultimately resulted in the deliberate killing of about 200,000 people with mental and/or physical disabilities.']], ['Memorandum Authorizing Involuntary Euthanasia', ['Adolf Hitler signed a memorandum authorizing involuntary euthanasia in October 1939 to serve as the legal basis for Aktion T4, the Nazi forced euthanasia program.', ' Its purpose was to assure the doctors and nurses who took part in the euthanasia program would not be prosecuted for murder.', ' During the postwar trials of these same individuals, they attempted to use this decree as a justification for their actions.']], ['Dasein ohne Leben', ['Dasein ohne Leben – Psychiatrie und Menschlichkeit (\"Existence Without Life\" – \"Psychiatry and Humanity\") is a 1942 Nazi propaganda film about the physically and mentally disabled: closeups of disabled persons.', ' The director was Hermann Schwenninger, one of the three managing directors of Gemeinnützige Krankentransport (\"Charitable Ambulance\"), a front company of Aktion T4, the central institution for the mass murder of patients in the Third Reich.', ' Schwenninger also wrote parts of the screenplay of \"Ich klage an\".', \" The contract for the film came from Hitler's Chancellery, and was produced by Tobis Film.\"]]], 'type': 'bridge', 'level': 'hard'}\n",
      "Error: The input to LLMOutputParser.parse should be a str, but found <class 'NoneType'>.\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  18%|█▊        | 90/500 [00:16<01:11,  5.72it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  18%|█▊        | 92/500 [00:16<01:17,  5.28it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  19%|█▊        | 93/500 [00:17<01:45,  3.87it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.6, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  19%|█▉        | 95/500 [00:17<01:18,  5.14it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  19%|█▉        | 96/500 [00:17<01:54,  3.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.27272727272727276, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  20%|█▉        | 99/500 [00:18<01:14,  5.42it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.19999999999999998, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.4, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  20%|██        | 101/500 [00:18<01:06,  5.97it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.1739130434782609, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.4, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  21%|██        | 103/500 [00:18<00:59,  6.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.1111111111111111, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  21%|██        | 104/500 [00:18<01:05,  6.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.5454545454545454, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  21%|██        | 106/500 [00:19<01:00,  6.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.9090909090909091, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  22%|██▏       | 109/500 [00:19<01:17,  5.07it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.5, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  22%|██▏       | 110/500 [00:20<01:11,  5.49it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.4615384615384615, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  22%|██▏       | 112/500 [00:20<01:02,  6.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  23%|██▎       | 113/500 [00:20<01:13,  5.27it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.3076923076923077, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.18181818181818182, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  23%|██▎       | 117/500 [00:21<00:53,  7.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  24%|██▎       | 118/500 [00:21<01:05,  5.80it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  24%|██▍       | 120/500 [00:21<01:07,  5.63it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.028169014084507043, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  25%|██▍       | 123/500 [00:22<00:53,  7.06it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  25%|██▌       | 125/500 [00:22<00:44,  8.47it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.7777777777777778, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.2857142857142857, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  26%|██▌       | 129/500 [00:23<00:56,  6.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  26%|██▌       | 130/500 [00:23<00:57,  6.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  26%|██▌       | 131/500 [00:23<01:23,  4.40it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  26%|██▋       | 132/500 [00:23<01:22,  4.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  27%|██▋       | 133/500 [00:24<01:25,  4.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.8571428571428571, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  27%|██▋       | 135/500 [00:24<01:04,  5.63it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  27%|██▋       | 136/500 [00:24<01:22,  4.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  29%|██▉       | 144/500 [00:25<00:30, 11.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.5714285714285715, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.33333333333333337, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  29%|██▉       | 146/500 [00:25<00:29, 12.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  30%|██▉       | 148/500 [00:26<01:00,  5.84it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.6, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  30%|███       | 152/500 [00:26<00:52,  6.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.42857142857142855, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  31%|███       | 155/500 [00:27<01:20,  4.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.14285714285714288, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  31%|███▏      | 157/500 [00:27<01:01,  5.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  32%|███▏      | 159/500 [00:28<01:04,  5.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  32%|███▏      | 162/500 [00:28<00:47,  7.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  33%|███▎      | 166/500 [00:29<00:53,  6.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  34%|███▍      | 170/500 [00:29<00:39,  8.40it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.5, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.625, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  35%|███▌      | 175/500 [00:30<00:47,  6.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  35%|███▌      | 177/500 [00:31<01:11,  4.49it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.5, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.1081081081081081, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.6, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  36%|███▌      | 180/500 [00:31<00:52,  6.07it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  37%|███▋      | 183/500 [00:32<00:52,  6.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  37%|███▋      | 185/500 [00:32<00:43,  7.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  37%|███▋      | 186/500 [00:32<00:45,  6.88it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.3076923076923077, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  38%|███▊      | 188/500 [00:32<00:44,  6.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  38%|███▊      | 190/500 [00:33<00:49,  6.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.08333333333333334, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  39%|███▊      | 193/500 [00:34<01:00,  5.06it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.28571428571428575, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  39%|███▉      | 194/500 [00:34<00:56,  5.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  39%|███▉      | 196/500 [00:34<01:00,  5.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.4, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  40%|████      | 200/500 [00:35<00:43,  6.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.14285714285714288, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  41%|████      | 204/500 [00:35<00:38,  7.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.5, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.4444444444444444, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  41%|████      | 206/500 [00:36<01:00,  4.83it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.7777777777777778, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  41%|████▏     | 207/500 [00:36<01:03,  4.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  42%|████▏     | 210/500 [00:37<00:48,  6.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.4, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  43%|████▎     | 213/500 [00:37<00:52,  5.48it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.8, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  43%|████▎     | 215/500 [00:38<00:51,  5.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.4, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  43%|████▎     | 217/500 [00:38<00:40,  7.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.2857142857142857, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  44%|████▍     | 219/500 [00:38<00:45,  6.17it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  44%|████▍     | 220/500 [00:39<00:47,  5.86it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  44%|████▍     | 222/500 [00:39<00:41,  6.66it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.21428571428571425, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  45%|████▌     | 225/500 [00:39<00:38,  7.07it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.15384615384615385, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  45%|████▌     | 226/500 [00:39<00:44,  6.09it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.6153846153846153, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  46%|████▌     | 228/500 [00:40<00:49,  5.55it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  46%|████▌     | 230/500 [00:40<00:58,  4.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.8, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.5, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  46%|████▋     | 232/500 [00:41<00:51,  5.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.5714285714285715, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  47%|████▋     | 234/500 [00:41<00:40,  6.52it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  47%|████▋     | 236/500 [00:41<00:37,  6.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  48%|████▊     | 238/500 [00:42<00:49,  5.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  48%|████▊     | 242/500 [00:42<00:34,  7.46it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.4, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  49%|████▉     | 245/500 [00:43<00:40,  6.27it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.14285714285714285, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  49%|████▉     | 246/500 [00:43<00:42,  6.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  50%|█████     | 250/500 [00:43<00:34,  7.29it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  50%|█████     | 252/500 [00:44<00:32,  7.58it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  51%|█████     | 254/500 [00:44<00:49,  4.92it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.3076923076923077, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.13333333333333336, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  51%|█████     | 255/500 [00:45<00:55,  4.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  51%|█████▏    | 257/500 [00:45<00:55,  4.38it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  52%|█████▏    | 260/500 [00:46<00:50,  4.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  53%|█████▎    | 263/500 [00:46<00:32,  7.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.5, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.09302325581395349, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  53%|█████▎    | 264/500 [00:46<00:34,  6.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.5454545454545454, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.5, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  53%|█████▎    | 266/500 [00:46<00:32,  7.11it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  53%|█████▎    | 267/500 [00:47<00:46,  5.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.5714285714285715, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.8, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  54%|█████▍    | 270/500 [00:47<00:37,  6.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  54%|█████▍    | 272/500 [00:47<00:36,  6.28it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  55%|█████▍    | 273/500 [00:48<00:41,  5.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  55%|█████▌    | 276/500 [00:48<00:33,  6.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.5625, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  55%|█████▌    | 277/500 [00:48<00:36,  6.17it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  56%|█████▌    | 279/500 [00:48<00:32,  6.74it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  56%|█████▋    | 282/500 [00:49<00:28,  7.77it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.2857142857142857, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  57%|█████▋    | 284/500 [00:49<00:24,  8.78it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  57%|█████▋    | 287/500 [00:50<00:33,  6.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.8571428571428571, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  58%|█████▊    | 289/500 [00:50<00:45,  4.60it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.23529411764705882, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.3333333333333333, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  58%|█████▊    | 292/500 [00:51<00:32,  6.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.2857142857142857, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  59%|█████▊    | 293/500 [00:51<00:37,  5.48it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  59%|█████▉    | 295/500 [00:51<00:32,  6.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  59%|█████▉    | 296/500 [00:51<00:42,  4.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.4, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  60%|█████▉    | 299/500 [00:52<00:42,  4.71it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.625, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  60%|██████    | 302/500 [00:52<00:29,  6.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.5, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  61%|██████    | 305/500 [00:53<00:29,  6.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  61%|██████    | 306/500 [00:53<00:34,  5.66it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.8, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  62%|██████▏   | 310/500 [00:54<00:34,  5.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.3571428571428571, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.34285714285714286, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.8, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  63%|██████▎   | 313/500 [00:54<00:29,  6.36it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  63%|██████▎   | 316/500 [00:55<00:19,  9.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.5, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  64%|██████▍   | 320/500 [00:55<00:23,  7.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.5714285714285715, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  64%|██████▍   | 322/500 [00:56<00:43,  4.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  65%|██████▌   | 325/500 [00:57<00:33,  5.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  66%|██████▌   | 328/500 [00:57<00:27,  6.17it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.5714285714285715, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  66%|██████▋   | 332/500 [00:58<00:24,  6.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.3157894736842105, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  67%|██████▋   | 335/500 [00:58<00:25,  6.40it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.5454545454545454, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  67%|██████▋   | 336/500 [00:58<00:29,  5.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  68%|██████▊   | 340/500 [00:59<00:22,  7.02it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.8, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  68%|██████▊   | 341/500 [00:59<00:26,  5.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.8, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  69%|██████▊   | 343/500 [00:59<00:23,  6.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.23529411764705882, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  69%|██████▉   | 346/500 [01:00<00:25,  6.09it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  70%|██████▉   | 349/500 [01:01<00:26,  5.78it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.8, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.3333333333333333, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  70%|███████   | 351/500 [01:01<00:33,  4.40it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.33333333333333337, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.4, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  70%|███████   | 352/500 [01:01<00:31,  4.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  71%|███████   | 356/500 [01:02<00:20,  7.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  72%|███████▏  | 358/500 [01:02<00:21,  6.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.8, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  72%|███████▏  | 359/500 [01:02<00:19,  7.11it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.5, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  72%|███████▏  | 361/500 [01:03<00:28,  4.82it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  72%|███████▏  | 362/500 [01:03<00:29,  4.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  73%|███████▎  | 364/500 [01:04<00:26,  5.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.8571428571428571, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.048780487804878044, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  73%|███████▎  | 365/500 [01:04<00:29,  4.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  73%|███████▎  | 367/500 [01:04<00:24,  5.32it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.8, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  74%|███████▎  | 368/500 [01:04<00:27,  4.72it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  74%|███████▍  | 370/500 [01:05<00:23,  5.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  75%|███████▍  | 374/500 [01:05<00:14,  8.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  75%|███████▌  | 375/500 [01:05<00:22,  5.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.125, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  75%|███████▌  | 377/500 [01:06<00:21,  5.72it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.4444444444444445, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  76%|███████▌  | 379/500 [01:06<00:20,  5.82it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  76%|███████▌  | 380/500 [01:06<00:22,  5.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.06451612903225806, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  77%|███████▋  | 384/500 [01:07<00:17,  6.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.75, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  77%|███████▋  | 385/500 [01:07<00:22,  5.16it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.6, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  78%|███████▊  | 388/500 [01:08<00:17,  6.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.5, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  78%|███████▊  | 389/500 [01:08<00:21,  5.28it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.4444444444444445, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  78%|███████▊  | 391/500 [01:08<00:17,  6.16it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  78%|███████▊  | 392/500 [01:09<00:24,  4.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  79%|███████▉  | 395/500 [01:09<00:18,  5.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.10526315789473684, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  79%|███████▉  | 396/500 [01:09<00:20,  5.04it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  80%|████████  | 400/500 [01:10<00:17,  5.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.5454545454545454, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  81%|████████  | 403/500 [01:10<00:12,  7.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.6363636363636364, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  81%|████████  | 405/500 [01:10<00:12,  7.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.41379310344827586, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534590a72d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458ff3e00>, 9882831.900972921)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534590a75d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458ee2490>\n",
      "Evaluating workflow:  81%|████████  | 406/500 [01:11<00:19,  4.88it/s]Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458f2cd90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534590c9fd0>, 9882838.278584408)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458f2cc50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345904a8d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534590c9240>, 9882832.884850688)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458ee2810>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458ff1ef0>, 9882831.427523242)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345904a790>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458f132d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534590cb0e0>, 9882833.9227552)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534595cd9d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458f75bd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458fc6cd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534590caf20>, 9882833.485776952)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458f759d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458fc06e0>, 9882833.419836726)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458fc7410>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458fb3b50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459302f90>, 9882833.734559556)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458fb3a10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534590838d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458eb1cc0>, 9882836.545759743)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458d08ed0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458cfe6d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458d9c3d0>, 9882838.642686889)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458cff3d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458ed0f10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458cf7810>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458cfe750>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458d40e50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534590c8c90>, 9882835.569353769)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458ed1590>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458eb1940>, 9882835.45715636)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458cf7d90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458ff2d60>, 9882835.484050402)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458cfe610>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458eb2900>, 9882835.978691092)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458d40c50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458d70e50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458eb2580>, 9882836.362148808)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458d70c50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458dce210>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458d9e9e0>, 9882839.550886922)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458dce390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458ea1790>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458d9f0e0>, 9882843.748531394)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458ea1b90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458c7fad0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458d9ff50>, 9882839.236639148)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458c7fd50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458d23f10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458c66fd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458c7d390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458ca8990>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458d9fcb0>, 9882839.608670356)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458c7d250>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458c9c520>, 9882839.61855446)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458c9b190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458d9d1d0>, 9882838.926174395)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458d23bd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458d9f770>, 9882839.920571996)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458c66c90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458b47ed0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458d9d9b0>, 9882842.418300232)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458b47dd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458b2fe10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458d9cd00>, 9882845.62442739)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458c7e110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534589fd050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458ff19b0>, 9882844.463479863)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534589fd250>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458a33d90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458c9d940>, 9882844.491282685)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458db1c10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458b77c90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458ba2450>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458bcc050>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534589d8ad0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458a27ad0>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458b782f0>, 9882842.748177644)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458b77b50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458b7b230>, 9882842.367578967)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458a275d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458fc3930>, 9882841.798520438)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458ba2310>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458b79240>, 9882842.813740004)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458bcc4d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458b79c50>, 9882842.62341245)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534589d9210>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458a6f950>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458a710f0>, 9882844.99469701)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458a45550>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458a96a90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458b79cc0>, 9882846.026654517)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458a96b50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534589f1050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345896de10>, 9882846.658695927)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534589f1290>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458b50c10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458903d90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458925d90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458a731c0>, 9882845.31032316)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458925c50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458a71e10>, 9882844.703264192)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458b510d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458b7a7b0>, 9882844.08250844)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458903c50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458c85f50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534588609f0>, 9882848.343938269)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534589620d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458b77590>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458a722e0>, 9882848.521123527)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458b77b10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534589610d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534588b17d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458b7a350>, 9882847.712289272)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458960d10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458d9da90>, 9882847.97851446)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534588b1410>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458864950>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458860670>, 9882848.871845668)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458864750>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534587b3d10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458745fd0>, 9882852.710383805)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534587b3c10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534587c9e50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458747d90>, 9882851.40044568)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534587cb190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345868d410>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345896ec80>, 9882852.974455036)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345868d610>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458673290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458b79550>, 9882854.090643408)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458673d90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534587972d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534585db3d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458745860>, 9882852.663472926)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458794f90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458862270>, 9882851.131142477)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534585d9110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534585ae8d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458536190>, 9882856.114402125)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534585ae850>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534586cbc10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458536cf0>, 9882855.69037116)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534586cbb50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534585fa810>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345863cf30>, 9882854.83743494)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534585f9550>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534584fd3d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345863f2a0>, 9882854.435514037)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458506dd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345865b450>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458434210>, 9882857.476258684)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345865b4d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534583ecb10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458436660>, 9882856.70534608)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534583ecbd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534585a0650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458200ad0>, 9882861.13784732)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458215690>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534585be2d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458201630>, 9882859.93803291)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534582398d0>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534585fa650>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534583ac5d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534584aee50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458744280>, 9882860.098073509)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458728550>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534583169e0>, 9882859.607659576)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534583ac450>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458435f60>, 9882859.847287528)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534581dc310>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534584aed10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345863c4b0>, 9882864.1207297)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534584af6d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458d4b590>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534580f3700>, 9882866.442518309)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534581b4250>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345815ca10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534580f1da0>, 9882873.879695881)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345815ca50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534581a29d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458314280>, 9882864.060176007)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534581a2fd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458136910>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458316270>, 9882862.26817764)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534581367d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153443f8f050>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153443f60290>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458233cd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153443f1a270>, 9882865.285610208)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153443f8ee50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153443f19a20>, 9882865.184962856)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153443f60550>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345863f700>, 9882866.705135787)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458233f50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345912cad0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153459154950>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345916ebd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153459171250>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534591c9390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345913c110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458feb9d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534591b0310>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534591b2250>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458ff7750>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153459015550>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153459081610>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534591ca390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153459185090>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534590d22d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153459322d90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458efee50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458fda690>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345904ab50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458f64990>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458f90c10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458f1f590>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458dedf90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458e03110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458e18850>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345904a850>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458e59310>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458e75e50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534590a6890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458e90dd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458e27b10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458d21b10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458d92850>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458dc02d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458ed2f10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458d89f50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458bee3d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458c118d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458c1a2d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458d93cd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458c9a250>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458cb2e50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458b08390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458b187d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458b44f10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458b5d8d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458bfd810>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458b5f110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458addb90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458a0fbd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458a4d610>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458a5f090>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458a868d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458a9d810>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458ab5410>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345894d9d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458960a90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345898c210>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534589a4dd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534589c5d50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534587e6450>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534588011d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345891c9d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345881a950>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534588676d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345888e990>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534586e75d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534586f42d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458702090>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534587fb610>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345897be10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534586e4390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534587f8ad0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345877dd10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534588b9cd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534585ed1d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534585eef90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534586115d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458750ed0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345864eed0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345865ac90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458699410>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534586af790>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534586c2d90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534585ee290>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458522a50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345854cad0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345857d7d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458505350>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534585bdbd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534583ef550>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458591fd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345841cf10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345857c050>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534583e7750>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458423190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458442f90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345857ed10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534584c6410>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534584cf510>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534582fe8d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534583191d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534583319d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345854c710>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345834f6d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458498850>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458479d90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345837a1d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345838d890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534583c00d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534583d1bd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458233810>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345826cb10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345826f410>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458295890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458299f10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534582a8d90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458256310>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534580e9750>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458118f10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534581296d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345837a5d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458191550>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534581a9e90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153443f15190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534581caed0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153443f49590>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458166450>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458181450>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345826f6d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345983fe70>, 9882830.367798489)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534592ccfd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534591f1550>, 9882831.371525154)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153459155c90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534590da820>, 9882829.125914356)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345916ea90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534591f0f30>, 9882830.484667726)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153459171150>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534590dbcb0>, 9882829.811011951)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534591c9190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534590dadd0>, 9882830.677291391)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345913c5d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458ff0520>, 9882831.22430714)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458feb450>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534590da0b0>, 9882829.879768025)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534591b0810>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534590db700>, 9882829.891834762)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534592c2990>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458ff07c0>, 9882836.544556811)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458ff5e10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458ff1550>, 9882833.004508631)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534590178d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534593010f0>, 9882833.306392608)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153459080750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458ff30e0>, 9882833.31641138)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534591ca4d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534590db4d0>, 9882833.027719624)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534591851d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534590c84b0>, 9882832.133219423)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534590c0b90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534590c8d00>, 9882832.533216286)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153459322e50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458ff0a60>, 9882832.69662018)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534592cee90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534590c89f0>, 9882833.009495305)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458fdac10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458ff3310>, 9882836.978533447)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345904a910>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534590cadd0>, 9882832.867393028)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458f65050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534590cb700>, 9882834.822920892)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153459061250>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534590cba10>, 9882836.11979208)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458f1fd90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458fc1010>, 9882834.238347083)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458dedbd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458fc1630>, 9882852.547752276)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153459293610>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458ff1b70>, 9882836.390956368)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458e1b950>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534591f3c40>, 9882838.008158552)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534590488d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534590c8670>, 9882835.396068484)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458e58e90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458fc3620>, 9882837.240396252)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458e75c90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458fc3f50>, 9882836.76737256)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458f2d6d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458fc3d90>, 9882836.702975493)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458e90c90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458ff00c0>, 9882836.604469024)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458f28a10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459302c10>, 9882839.639974916)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458d21910>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458eb08a0>, 9882838.504123325)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458d92650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458fc3460>, 9882838.289261768)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458dc0790>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458eb0f30>, 9882858.547463996)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458ed3190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458fc0590>, 9882842.918526353)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458d89d50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534591f0520>, 9882838.132578876)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458bee290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534590cb310>, 9882839.134384302)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153459081d50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534590d8d00>, 9882839.7768469)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458c1a190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458d9c830>, 9882841.637364028)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458d93d90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458c9d6a0>, 9882840.540892938)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458cd3190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458c9dc50>, 9882841.25154344)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458e1b190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458c9de10>, 9882841.157439565)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458af11d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458c9e7b0>, 9882840.521569893)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458b1ad10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534590c9e80>, 9882842.516898043)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458b45650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458c9f9a0>, 9882843.18295312)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458b5e010>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458d9e820>, 9882840.614111474)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534591cbb90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458c9fd90>, 9882841.54491216)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458b5f9d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458d9f620>, 9882845.939949809)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458addc90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458b7aba0>, 9882842.735971356)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458a0fa90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458b7bb60>, 9882844.893009888)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458a4e050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458b7bf50>, 9882845.763164388)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458a5eb50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458a70830>, 9882844.389407497)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458a86e50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458a70de0>, 9882845.63025631)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458a9d6d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458a71a90>, 9882846.544608936)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458dcd1d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458a73af0>, 9882846.367566925)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345894dd90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345896c2f0>, 9882845.806240644)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458953190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345896cb40>, 9882848.017825903)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345898c8d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345896d2b0>, 9882848.164142264)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534589a4bd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458b7a740>, 9882847.961253632)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534589c5c10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345896e040>, 9882850.184232617)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534587e6b50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345896e970>, 9882846.966827348)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458801a10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458a73000>, 9882846.695345648)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345891d290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534590caac0>, 9882848.281656545)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345881a810>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534588604b0>, 9882850.301262196)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534588674d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458a72ac0>, 9882850.081019595)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345888ee50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458a71780>, 9882850.340481183)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534586e7d10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458b78fa0>, 9882848.89417808)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458962050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458863150>, 9882849.067896636)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458701f50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458a73b60>, 9882849.348733516)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458803290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345896c830>, 9882850.123670006)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534589a6090>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345896d7f0>, 9882849.458319815)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534586dffd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458744fa0>, 9882849.976969851)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534587f8b50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534587451d0>, 9882852.970568301)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345877db10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458745be0>, 9882852.756527748)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534587e6e50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458862660>, 9882851.524925016)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534585ecd50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458745b70>, 9882851.36258266)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534585ef1d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534587475b0>, 9882852.208805691)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534586113d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534587477e0>, 9882852.38886576)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458750fd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458746f90>, 9882856.552408598)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345864ed90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345863cd70>, 9882852.6346298)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345865ab50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345863da90>, 9882852.173985152)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458961450>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345863e2e0>, 9882855.17999348)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534586af350>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345863ec10>, 9882853.290288351)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534586c0e90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345863f620>, 9882855.752354112)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534585ed250>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458534050>, 9882855.69870137)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458522690>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534585348a0>, 9882856.025856001)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345854cfd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458535a20>, 9882854.846533272)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345857de90>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458745400>, 9882855.398519088)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534585045d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458536740>, 9882858.665255833)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534585be290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534585374d0>, 9882855.564027146)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534583ef410>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534585379a0>, 9882855.741857)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458592bd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458537b60>, 9882857.849587863)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345841cd10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458747230>, 9882855.066393744)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345857c890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345863ee40>, 9882855.74778936)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534583e7610>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458434050>, 9882856.518196588)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458422dd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458434c90>, 9882860.676346233)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458442e50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534584343d0>, 9882857.350065542)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345857ea10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534587466d0>, 9882860.801763875)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534584c6790>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458437310>, 9882857.86699012)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534584cf310>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458747cb0>, 9882857.401718596)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534582fee50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458860d70>, 9882860.62367409)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458319890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458315010>, 9882859.515757697)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458331810>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458315240>, 9882858.031064808)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534584c7e50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458315400>, 9882858.30883852)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345834f4d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534584364a0>, 9882857.737940295)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458d7fdd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458314600>, 9882857.783762977)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345847a490>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458435da0>, 9882860.98114276)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345837a050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534583165f0>, 9882860.617244668)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345838d6d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458537ee0>, 9882860.646632558)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458f67350>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458437ee0>, 9882862.10548674)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534583d1c10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458201010>, 9882861.217488589)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458231850>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458201e10>, 9882861.914757792)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345826c910>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534582000c0>, 9882862.581165327)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458fa7d50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458202740>, 9882861.39330765)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458295690>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534582029e0>, 9882886.055100188)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458332550>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458201470>, 9882863.62616032)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534582a8c50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458201be0>, 9882862.03502244)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345848d550>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458203e70>, 9882865.116673896)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534580e9550>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458317e00>, 9882867.651596362)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458118d10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458434b40>, 9882866.101532647)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534581294d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534580f1860>, 9882864.3882452)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345837a2d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458202430>, 9882862.831469087)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458191410>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458316660>, 9882865.626735162)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458272450>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153443f18210>, 9882863.662269572)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153443f14f90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534580f0de0>, 9882865.35721851)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534581caf50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153443f19240>, 9882867.602738757)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153443f49450>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534580f02f0>, 9882863.739758171)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458166a90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534580f27b0>, 9882864.336323071)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458181410>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458201e80>, 9882864.186263744)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345826f4d0>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345900af90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458803590>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153443f71f10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459300130>, 9882830.630989872)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345900b690>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345896f460>, 9882848.748650262)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458803990>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153443f19cc0>, 9882864.882146614)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153443f719d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458a4ef10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458ba9810>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458bc47d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534589f0450>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458b2e410>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458fd3e90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534592b7190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345977b1d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534595183d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345958bf10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153459722c50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153459a08690>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153459ae1f10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459f8e580>, 9883849.362887615)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1533fac65a50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459f8f460>, 9883849.46449386)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153460351c10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459f8dfd0>, 9883849.77909316)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534587ebb10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459f8ce50>, 9883849.852011088)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458bc58d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459f8f770>, 9883850.01806002)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534589f0d10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459f8c6e0>, 9883850.533831237)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458b2f950>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459f8e120>, 9883851.22707024)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458fd3510>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459f8e430>, 9883851.42781599)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534592b71d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459f8d240>, 9883849.066391263)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345977b090>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459f8d080>, 9883851.30401605)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153459518b50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459d74360>, 9883848.712876832)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153459588250>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459f8cc90>, 9883854.070222097)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534598d2290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459f8da90>, 9883849.952562304)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153459a09fd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459f8fb60>, 9883849.829261934)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153459b713d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345a0800d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345a0c89d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345a1d1dd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345a09cb50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534596c4a10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459f8d5c0>, 9883850.284753729)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345a09ea50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345a013210>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459f8e5f0>, 9883849.571961029)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153459e57b10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459f8eac0>, 9883850.077623246)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345a0c8510>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459f8f070>, 9883849.539761344)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345a1d1710>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459f8fbd0>, 9883849.481346363)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534596c4710>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459f8fd90>, 9883850.46309058)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153459f43810>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345841e510>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459b58440>, 9883851.363755915)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345a2fff90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153459277850>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459d77770>, 9883851.123304488)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458a46850>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534590c2f90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345a330e90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459b589f0>, 9883855.602370575)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534589f06d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a5f2510>, 9883851.381485393)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345a330710>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345a323b50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345a568d10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a5f0360>, 9883852.987619191)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345a320c90>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459f8e6d0>, 9883851.42422691)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345a599ad0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345a550650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a5f1630>, 9883853.400218781)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345a551010>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345a0e1350>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345a471250>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a5f0210>, 9883852.615392197)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345a0e0890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a5f2200>, 9883852.64354076)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345a59bf50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153459b721d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345a42df10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a5f3d20>, 9883853.04456058)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153459ae20d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345a41a490>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345a5c2650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345983d0f0>, 9883852.355189033)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345a42e790>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345aabab30>, 9883852.696989477)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345a41add0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a5f1780>, 9883853.946893793)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345a5c3b90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345a7c6490>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345a6be010>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345bae31c0>, 9883854.358604085)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345a7c6e50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459f8d710>, 9883854.847490994)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345a6bea10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345a629810>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458fc3310>, 9883855.26945972)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345a62a390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345a52ddd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345ad5cfa0>, 9883857.946523728)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345a09ec50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345a672450>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345a797990>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345ad5c1a0>, 9883854.128044246)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345a670750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345ad5c980>, 9883855.390472023)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458af1650>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345a780410>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345ad5f930>, 9883869.053328088)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153459218fd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345a95f550>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345ad5c0c0>, 9883855.465034852)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534596c6e90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345a722b50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345a853190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345ad5eeb0>, 9883855.47299862)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345a720b10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345a8bd710>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345ad5edd0>, 9883855.461567327)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345a8bd410>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345ad77610>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345a7da610>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345ad5e200>, 9883859.026595611)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345ad74390>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345ad5dc50>, 9883856.054972436)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345a7da250>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345a838050>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345a6a7f50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458fa4cd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345a2c2090>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345ad5d630>, 9883855.389149357)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345a6dc4d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345ad5f460>, 9883855.820167545)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345a362790>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345bae2cf0>, 9883852.479217464)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345a6a7650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a8db1c0>, 9883853.833369948)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458f3ff50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a5f3cb0>, 9883853.73377564)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153459518290>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345a4f7c90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459f8f310>, 9883856.670646897)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345a4f76d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345a918c90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1533fae1a510>, 9883857.868799925)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345a918310>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345a9f3c50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1533fae19a90>, 9883856.67239898)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345a9f37d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345a928dd0>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1533fae19240>, 9883856.659241436)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345a928210>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534596a0750>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1533fadd9e10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a5f2c10>, 9883858.512184376)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534596a2590>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345bae2eb0>, 9883856.63370036)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1533fadd8f10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345ad10750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345ad5f2a0>, 9883857.623085132)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345ad0fc90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345aa00250>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15346030b930>, 9883858.221275045)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345a702050>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345a966f90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345a9bc650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15358ea0c9f0>, 9883859.931994876)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345a647d90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345941e820>, 9883858.097406248)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1533fadd5c10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345a9be110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1533fad7e3d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345941e270>, 9883861.281571547)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1533fade45d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1533fada4a10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345941e430>, 9883858.535930185)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1533fada6fd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1533faffd150>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345a896d90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1533fae1b5b0>, 9883855.983257772)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345a5c3190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345941fe00>, 9883857.751291564)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1533fad7e310>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345941eeb0>, 9883858.456012012)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1533faffed90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345f58bb50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345941fa80>, 9883857.594234912)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345f58b8d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345d6708d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1533fae18210>, 9883859.590555202)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15346034b190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345a9e2890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345d68abd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345941e040>, 9883861.914793031)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345a9e2bd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459f8e2e0>, 9883859.357506672)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345d68a990>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345d665250>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534590d99b0>, 9883859.318509392)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345d6663d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1533fad76d10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534590d8c20>, 9883860.541691393)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1533fad76910>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345d6ae890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534591f1160>, 9883860.68340071)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345d6ad4d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345f66ea50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534591f0ec0>, 9883859.869623508)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345f66fe90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345d701f10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345d525ed0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459d763c0>, 9883860.109149503)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345d702090>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534591f3e00>, 9883862.456284696)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345d526a50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345a52f210>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345d53fc90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a5f12b0>, 9883860.432861332)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345a52f310>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534591f08a0>, 9883860.287306588)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345d53fa50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345d55d210>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345f66edd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345d652690>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458a70fa0>, 9883861.091053547)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345d55cf10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15346030af90>, 9883858.276078012)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345f66ec50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345ad5de10>, 9883858.322472775)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345d652350>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345d581810>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534590d91d0>, 9883862.98103558)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345d581650>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345d593ed0>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458ff10f0>, 9883861.992317837)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345d593c90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345d5ac350>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534591f3b60>, 9883861.109991845)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345d5ac790>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345a9b1450>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345941e190>, 9883863.152682845)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345d6c8bd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345d5d7790>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459513540>, 9883861.920519987)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345d5d7b10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345d6be350>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345d60f810>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459511860>, 9883862.06981597)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345d700b90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345d430c90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534595134d0>, 9883863.392150424)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345a8314d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345d43c910>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1533fae1a5f0>, 9883868.891983386)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345d43c550>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345d4433d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345d467e90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459b580c0>, 9883862.374583762)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345a52e810>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345962ee40>, 9883864.049968235)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345d467c90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345d47fed0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345962f700>, 9883862.71506101)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345d47f990>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345d4cc590>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345a734150>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345d56eed0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345d5c2290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459513a10>, 9883877.495650845)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345d60f610>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459512c80>, 9883861.346780324)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345d5c20d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345941f700>, 9883865.565493697)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345d4cd810>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345941f690>, 9883861.924181644)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345a7341d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458ff0750>, 9883861.596849415)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345d56ecd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345d456950>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345962f8c0>, 9883864.882207826)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345a942210>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345d4f7ed0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345962c0c0>, 9883864.02880185)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345d4f7dd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345d32e610>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345962ff50>, 9883865.344861267)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345d32ed50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345d35f650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345962d860>, 9883864.8364415)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345d60fc90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345d5685d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459512cf0>, 9883868.545652373)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345d568250>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345d3782d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345973b9a0>, 9883865.43022344)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345d378790>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345d4a0190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345d38a890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345973aba0>, 9883865.13001909)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345ad311d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345973ae40>, 9883864.821268056)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345d38a690>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345d399c10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345973a430>, 9883865.081122562)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345d399ad0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345d3aca90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534597399b0>, 9883866.062277464)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345a463a90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345d390a50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345d3cb050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345973ba80>, 9883865.810902536)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345d390ed0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345d3ce6d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345973b310>, 9883866.978464944)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345d3cee50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345d3e2ad0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153459eb2010>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345d36ac10>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458ff3930>, 9883865.7958848)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345d3cb550>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345962d710>, 9883867.226814639)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345d3e2fd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345962c280>, 9883863.579615075)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153459eb39d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345962c7c0>, 9883864.249071704)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345d36aa10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345d46cf10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458ff3770>, 9883866.985894512)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345d5d7cd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345d417c50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345962fe00>, 9883867.568500508)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345d414550>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345d41df50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345993ca60>, 9883868.92898061)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345d41de10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345bbdac90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345993d0f0>, 9883871.074654723)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345bbda8d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345d40b010>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345bbef610>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458d9c0c0>, 9883867.525371438)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345d81e810>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345bbf6ad0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345993cec0>, 9883867.884729648)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345bbf6e50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345d3ee790>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345bc0b5d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345962d1d0>, 9883869.25146002)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345d379ed0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459a38280>, 9883867.856202101)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345bc0b490>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345bc15e50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459a3ac80>, 9883867.985611025)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345bc16510>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345bc17fd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345bc267d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345962c3d0>, 9883868.679785144)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345bc17b10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459a3a6d0>, 9883868.28774234)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345bc26e90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345d3ec410>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345bbc7650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458ff1be0>, 9883869.12661705)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345bbef410>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345d406050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534597387c0>, 9883866.100529375)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345d3ee250>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345993fb60>, 9883866.703035831)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345bbc7450>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459513e00>, 9883866.851448992)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345d431c10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345bc315d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459a3a970>, 9883877.660464885)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345bc31c50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345d40fa10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345bc7ba10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459a3bc40>, 9883869.821326602)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345d40ff50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459a3be00>, 9883870.6010231)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345bc7b8d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345bc96a90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459a3acf0>, 9883870.122139636)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345bc962d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345bbfba90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459738a60>, 9883871.668116655)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345bbfb190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345bcae210>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345bcbaed0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459c5e6d0>, 9883870.820193816)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345bcae0d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534595109f0>, 9883871.982261393)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345bcbacd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153443120510>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345993e6d0>, 9883871.130505145)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344311af90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153443120410>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459c5e200>, 9883871.957915325)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153443121190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344312da50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345ad5d860>, 9883872.054017004)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153443109c10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344312f250>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153443151490>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345993d710>, 9883871.732824553)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344312d650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459c5d8d0>, 9883872.048699256)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153443151b90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153443110a50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459c5dfd0>, 9883872.016599173)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344315b190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345bbc74d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345bc68a50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459738d70>, 9883869.211284088)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345d3e37d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459a3bbd0>, 9883869.245605491)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345bc69110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345bc8db90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345993f700>, 9883869.95929484)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345bc8de90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459511f60>, 9883871.9490929)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153443110b50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153443170150>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534431ba910>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459c5e0b0>, 9883873.074752325)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153443171410>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a19be70>, 9883873.532437539)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534431ba7d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534431d1410>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345993d780>, 9883873.042375982)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534431d1210>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534431ea010>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a199780>, 9883874.923016954)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534431ea650>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153443012310>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153443118390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153443029f10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459c5cb40>, 9883873.438914677)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153443118650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a0927b0>, 9883873.616519678)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153443029dd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153443181a50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345bc9d750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459a3bcb0>, 9883875.759797977)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534431babd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a093af0>, 9883874.823078737)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345bc9d2d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344318e050>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153443077fd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153443176090>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153443171d90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344318e010>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534431a5450>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459a39be0>, 9883873.198037554)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153443012110>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a199240>, 9883872.092353657)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344319ee50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a19a6d0>, 9883872.211928232)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345bbc7e50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a090b40>, 9883874.115875892)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344318ee50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459c5ed60>, 9883876.285346184)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153443077ed0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459a38d70>, 9883871.69424784)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153443176790>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a19ac80>, 9883871.279823)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344318e710>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153443085d50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459c5e3c0>, 9883876.405022824)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534430853d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344303a910>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a0911d0>, 9883875.336004755)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344303b6d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442f03150>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459a38bb0>, 9883876.530942973)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442f034d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153443055510>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442f1ca10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345993c590>, 9883876.400675919)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153443056bd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459e74910>, 9883876.250240894)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534430d4fd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534430f6cd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459e746e0>, 9883875.609879144)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534430f7d50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534430e2c50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442f46610>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459e748a0>, 9883876.719585368)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534430e2d90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459c5ec10>, 9883876.8704342)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442f46d50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442f4ed10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459e76e40>, 9883876.5928175)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442f4ebd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442f86c10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442f9b9d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344307fdd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344309b810>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534430b2250>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153443016190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534430d67d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a0935b0>, 9883877.463587523)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442f87350>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a198130>, 9883875.494053496)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344309ae10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a092c10>, 9883875.234184232)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153443016210>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345ad5d1d0>, 9883879.239422165)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442f9b890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a093770>, 9883874.66167274)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344307fc90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a091fd0>, 9883875.471925803)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534430b2110>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a090210>, 9883874.792627366)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534430d6e90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534430b2550>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459e77000>, 9883880.86973719)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534430b23d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534431e2dd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459e74a60>, 9883877.525021683)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345d5aee50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442f8dd90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442fb4350>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a198750>, 9883878.542953927)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442fca810>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a2b1080>, 9883880.220225845)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442fd7f90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442fef290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a0937e0>, 9883878.593561426)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442fef810>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442ff11d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442e09d50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a2b0750>, 9883879.035588907)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442ff13d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a2b2f90>, 9883878.406462269)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442e0a590>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442e24890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a0908a0>, 9883879.546239076)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442e24690>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442e38990>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442f1c7d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a2b0590>, 9883881.05845917)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442f68050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a2b3310>, 9883878.518842245)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442f1d250>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345d3d2d50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442e67f50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a2b1630>, 9883878.859043049)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153443087690>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a2b1160>, 9883878.716016956)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442e67e50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534431e1250>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442e82ed0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a2b38c0>, 9883879.214836383)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442e71c10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a2b0c20>, 9883879.57089372)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442e82cd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442e94e10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459a3b150>, 9883879.370571708)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442e94c10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442ead5d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a4c9400>, 9883879.842866253)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442eac290>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442ebfb50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442ed9610>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a4c99b0>, 9883882.650525456)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442ebff90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a4c9f60>, 9883881.186949164)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442ed9cd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442ee4690>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a092ba0>, 9883881.643986708)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442ee47d0>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345bcb4d50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442ff0410>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a2b2900>, 9883881.224946825)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344309b890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a4ca900>, 9883881.980106764)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442ff04d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442eeecd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442d3efd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a4c9160>, 9883884.732017431)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442eef950>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a4ca510>, 9883882.579265784)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442d3f6d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442d4ab90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a4cb930>, 9883882.583317945)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442d4aa50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442d22350>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442d74910>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a2b17f0>, 9883880.960535683)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442d21350>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a3b5940>, 9883881.967822172)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442fa1bd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442d802d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442d9b890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a3b63c0>, 9883882.065750545)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442d81f10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a3b7310>, 9883885.723241812)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442d9b690>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442db2bd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a3b6200>, 9883883.734724348)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442db3310>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442e95050>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442dc71d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442ebdf90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a3b79a0>, 9883882.12856295)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442e95d90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a3b6040>, 9883885.684897)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442dc4850>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a4c8d70>, 9883879.823369976)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442ebe650>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442dd2090>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442de5cd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a3b5710>, 9883884.029810842)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442de5910>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442d77f90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a4cbd20>, 9883884.834357588)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153443047590>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442c48910>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459e76c10>, 9883883.480035037)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442c48750>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442c12790>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442c57290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a2b15c0>, 9883888.425742673)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442c12f50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a4c9940>, 9883884.11027442)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442c57150>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442c95990>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a3e2660>, 9883885.31586289)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442c95850>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442de9810>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442cb5d50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a3e1be0>, 9883885.183701837)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442deab90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a3e0050>, 9883885.100548504)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442cb6410>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442c1ef90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442cd10d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a3e1470>, 9883885.24619112)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442c1e8d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a092c80>, 9883887.19604382)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442cd15d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442cf5b10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a6f3a10>, 9883887.64605254)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442cf53d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442c85e10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442910910>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a2b04b0>, 9883883.959761718)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442d47890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442e64610>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442c03550>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a092660>, 9883882.69869134)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442c03350>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a6f2cf0>, 9883884.728115184)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442ec8210>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a2b2c10>, 9883886.199227097)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442910710>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a3e3540>, 9883883.380188229)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442c26550>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344291d9d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344293a6d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a6f0590>, 9883886.154766804)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344291e110>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a6f0f30>, 9883886.375482732)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344293add0>\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.3333333333333333, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.8, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.13333333333333333, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  82%|████████▏ | 410/500 [01:12<00:17,  5.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.3, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.5, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  83%|████████▎ | 414/500 [01:12<00:12,  6.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  83%|████████▎ | 416/500 [01:12<00:13,  6.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.5, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  84%|████████▍ | 421/500 [01:13<00:10,  7.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.4, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.3157894736842105, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.8, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  85%|████████▍ | 423/500 [01:13<00:09,  8.11it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  85%|████████▍ | 424/500 [01:14<00:15,  4.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.8571428571428571, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  85%|████████▌ | 427/500 [01:14<00:11,  6.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.5, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  86%|████████▌ | 429/500 [01:14<00:11,  6.14it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.5, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  86%|████████▌ | 431/500 [01:15<00:11,  6.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.19999999999999998, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.6, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  87%|████████▋ | 435/500 [01:15<00:08,  7.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.2857142857142857, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  87%|████████▋ | 436/500 [01:15<00:08,  7.14it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.4444444444444445, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  88%|████████▊ | 441/500 [01:17<00:10,  5.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.8235294117647058, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  89%|████████▊ | 443/500 [01:17<00:08,  6.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.5454545454545454, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.4444444444444445, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  89%|████████▉ | 445/500 [01:17<00:07,  7.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.0909090909090909, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.5, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  89%|████████▉ | 447/500 [01:17<00:06,  7.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  90%|████████▉ | 449/500 [01:17<00:07,  7.06it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  91%|█████████ | 453/500 [01:18<00:08,  5.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.6153846153846153, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  91%|█████████ | 456/500 [01:19<00:06,  7.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.5263157894736842, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.5, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  92%|█████████▏| 460/500 [01:20<00:07,  5.36it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  92%|█████████▏| 462/500 [01:20<00:05,  6.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  93%|█████████▎| 467/500 [01:20<00:03, 10.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.2857142857142857, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  94%|█████████▍| 469/500 [01:20<00:02, 11.40it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  94%|█████████▍| 471/500 [01:21<00:05,  5.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.5, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  95%|█████████▍| 473/500 [01:22<00:04,  5.77it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.47619047619047616, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  95%|█████████▌| 475/500 [01:22<00:05,  4.72it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  96%|█████████▌| 479/500 [01:23<00:03,  6.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.4, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  96%|█████████▌| 481/500 [01:23<00:02,  6.89it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.4444444444444445, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  97%|█████████▋| 483/500 [01:23<00:02,  6.04it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.5, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  97%|█████████▋| 484/500 [01:23<00:02,  6.09it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.3333333333333333, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  97%|█████████▋| 485/500 [01:24<00:03,  4.48it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.8, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  97%|█████████▋| 487/500 [01:24<00:03,  4.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.375, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  98%|█████████▊| 489/500 [01:25<00:01,  5.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.5714285714285715, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  98%|█████████▊| 490/500 [01:25<00:01,  5.48it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.10526315789473684, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.33333333333333337, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  99%|█████████▉| 494/500 [01:25<00:00,  6.09it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.5714285714285715, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.33333333333333337, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  99%|█████████▉| 495/500 [01:26<00:00,  6.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.8, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow: 100%|█████████▉| 498/500 [01:26<00:00,  4.88it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.2857142857142857, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow: 100%|█████████▉| 499/500 [01:27<00:00,  4.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow: 100%|██████████| 500/500 [01:30<00:00,  5.54it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.044444444444444446, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   1%|          | 3/500 [00:02<04:42,  1.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   1%|          | 5/500 [00:02<02:38,  3.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.3333333333333333, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   2%|▏         | 8/500 [00:02<01:33,  5.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.8, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   2%|▏         | 11/500 [00:02<01:24,  5.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   3%|▎         | 15/500 [00:03<01:05,  7.37it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.75, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   3%|▎         | 16/500 [00:03<01:11,  6.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   4%|▎         | 18/500 [00:04<01:44,  4.63it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   4%|▍         | 21/500 [00:04<01:11,  6.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.4, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   4%|▍         | 22/500 [00:04<01:23,  5.71it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.7272727272727273, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   5%|▍         | 23/500 [00:04<01:26,  5.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   6%|▌         | 29/500 [00:05<00:53,  8.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.18181818181818182, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   6%|▌         | 31/500 [00:06<01:10,  6.63it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.3333333333333333, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   7%|▋         | 33/500 [00:06<01:21,  5.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   7%|▋         | 34/500 [00:06<01:24,  5.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   7%|▋         | 35/500 [00:07<01:38,  4.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.4827586206896552, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   7%|▋         | 37/500 [00:07<01:34,  4.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   8%|▊         | 38/500 [00:07<01:38,  4.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.3333333333333333, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   8%|▊         | 40/500 [00:08<01:44,  4.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   8%|▊         | 42/500 [00:08<01:36,  4.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   9%|▊         | 43/500 [00:08<01:31,  4.97it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   9%|▉         | 46/500 [00:09<01:10,  6.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.19999999999999998, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  10%|█         | 50/500 [00:09<00:43, 10.46it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.4444444444444445, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.13333333333333333, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  11%|█         | 55/500 [00:09<00:45,  9.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.5, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.33333333333333337, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.4444444444444445, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  12%|█▏        | 58/500 [00:11<01:41,  4.36it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  13%|█▎        | 63/500 [00:12<01:15,  5.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.125, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  14%|█▎        | 68/500 [00:12<01:02,  6.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  14%|█▍        | 70/500 [00:13<00:57,  7.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.5, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  15%|█▍        | 73/500 [00:14<01:27,  4.87it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.8, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.33333333333333337, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  15%|█▍        | 74/500 [00:14<01:21,  5.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  15%|█▌        | 77/500 [00:14<01:07,  6.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  16%|█▌        | 79/500 [00:14<00:55,  7.63it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.3333333333333333, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  16%|█▌        | 81/500 [00:14<00:50,  8.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  17%|█▋        | 83/500 [00:15<01:03,  6.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  17%|█▋        | 85/500 [00:15<01:10,  5.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.16666666666666669, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.5454545454545454, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  17%|█▋        | 86/500 [00:16<01:12,  5.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.10526315789473684, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  17%|█▋        | 87/500 [00:16<01:38,  4.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  18%|█▊        | 91/500 [00:16<00:58,  7.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.6, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  18%|█▊        | 92/500 [00:17<01:39,  4.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  19%|█▊        | 93/500 [00:17<01:36,  4.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  20%|██        | 100/500 [00:18<01:02,  6.36it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.28571428571428575, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  20%|██        | 102/500 [00:18<00:59,  6.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.3636363636363636, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  21%|██        | 104/500 [00:19<00:56,  7.06it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  21%|██        | 106/500 [00:19<01:01,  6.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  22%|██▏       | 108/500 [00:19<00:50,  7.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.19999999999999998, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  22%|██▏       | 110/500 [00:20<00:51,  7.60it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.9090909090909091, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.4615384615384615, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  22%|██▏       | 112/500 [00:20<01:12,  5.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.4615384615384615, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  23%|██▎       | 114/500 [00:20<00:59,  6.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.5, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  23%|██▎       | 115/500 [00:21<01:04,  5.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.2857142857142857, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  24%|██▍       | 120/500 [00:21<00:52,  7.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.8, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  24%|██▍       | 122/500 [00:21<00:47,  7.96it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.3076923076923077, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  25%|██▍       | 124/500 [00:22<00:44,  8.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  26%|██▌       | 128/500 [00:22<00:44,  8.28it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.7777777777777778, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  26%|██▌       | 130/500 [00:23<00:52,  7.04it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.2857142857142857, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  26%|██▌       | 131/500 [00:23<01:16,  4.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.05, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  27%|██▋       | 133/500 [00:24<01:16,  4.83it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  27%|██▋       | 134/500 [00:24<01:30,  4.05it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  27%|██▋       | 137/500 [00:24<01:02,  5.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  28%|██▊       | 142/500 [00:25<00:37,  9.66it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  29%|██▉       | 144/500 [00:25<00:51,  6.89it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.21052631578947367, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  30%|██▉       | 148/500 [00:25<00:37,  9.27it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.7692307692307693, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.6, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  30%|███       | 150/500 [00:26<01:21,  4.28it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  30%|███       | 152/500 [00:27<01:40,  3.46it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  31%|███       | 156/500 [00:28<01:04,  5.32it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  32%|███▏      | 158/500 [00:28<00:50,  6.80it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  32%|███▏      | 160/500 [00:28<00:46,  7.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  32%|███▏      | 162/500 [00:28<00:58,  5.74it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  33%|███▎      | 164/500 [00:29<00:53,  6.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.14285714285714288, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  33%|███▎      | 166/500 [00:29<00:53,  6.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  34%|███▎      | 168/500 [00:29<00:53,  6.16it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.5, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  34%|███▍      | 169/500 [00:30<00:56,  5.88it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  34%|███▍      | 171/500 [00:30<00:48,  6.78it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  35%|███▌      | 176/500 [00:31<00:42,  7.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.7142857142857143, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.22222222222222224, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.25, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  36%|███▌      | 178/500 [00:32<01:11,  4.49it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  37%|███▋      | 184/500 [00:32<00:38,  8.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.23076923076923078, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  38%|███▊      | 189/500 [00:33<00:43,  7.11it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.3076923076923077, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.4, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  38%|███▊      | 191/500 [00:33<00:36,  8.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  39%|███▊      | 193/500 [00:33<00:38,  8.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  39%|███▉      | 195/500 [00:34<01:04,  4.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  39%|███▉      | 197/500 [00:34<00:55,  5.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  40%|███▉      | 199/500 [00:35<00:53,  5.58it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.38095238095238093, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  40%|████      | 202/500 [00:35<00:38,  7.80it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.19999999999999998, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  41%|████      | 204/500 [00:35<00:51,  5.72it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  41%|████      | 206/500 [00:36<00:50,  5.82it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.7777777777777778, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  42%|████▏     | 208/500 [00:36<00:57,  5.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.4, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  42%|████▏     | 210/500 [00:37<00:47,  6.04it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  42%|████▏     | 212/500 [00:37<01:19,  3.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.8, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  43%|████▎     | 214/500 [00:38<01:20,  3.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.4, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  43%|████▎     | 216/500 [00:38<00:53,  5.35it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  44%|████▍     | 219/500 [00:39<00:40,  6.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.6153846153846153, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  44%|████▍     | 222/500 [00:39<00:32,  8.67it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.21428571428571425, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  45%|████▌     | 226/500 [00:39<00:23, 11.60it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.15384615384615385, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  46%|████▌     | 228/500 [00:39<00:30,  8.84it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.5, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  46%|████▌     | 230/500 [00:40<01:01,  4.42it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.5, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  47%|████▋     | 233/500 [00:41<00:51,  5.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  47%|████▋     | 236/500 [00:41<00:43,  6.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  48%|████▊     | 240/500 [00:42<00:34,  7.48it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  49%|████▊     | 243/500 [00:42<00:40,  6.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  49%|████▉     | 245/500 [00:43<00:51,  4.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  50%|████▉     | 248/500 [00:43<00:33,  7.60it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.14285714285714288, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.14285714285714285, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.4, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  50%|█████     | 252/500 [00:44<00:30,  8.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  51%|█████     | 255/500 [00:44<00:21, 11.36it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.11764705882352941, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.4444444444444445, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  51%|█████▏    | 257/500 [00:44<00:39,  6.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.08333333333333333, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  52%|█████▏    | 261/500 [00:45<00:32,  7.47it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.5454545454545454, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.5714285714285715, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  53%|█████▎    | 263/500 [00:46<00:50,  4.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  53%|█████▎    | 266/500 [00:47<00:57,  4.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.6, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.18181818181818182, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  54%|█████▍    | 271/500 [00:47<00:29,  7.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.2857142857142857, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  55%|█████▍    | 274/500 [00:48<00:34,  6.49it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  55%|█████▌    | 276/500 [00:48<00:31,  7.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  55%|█████▌    | 277/500 [00:48<00:44,  5.07it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.6, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  56%|█████▌    | 279/500 [00:49<00:42,  5.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  56%|█████▌    | 280/500 [00:49<00:42,  5.14it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  56%|█████▋    | 282/500 [00:49<00:36,  5.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  57%|█████▋    | 285/500 [00:49<00:28,  7.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.5454545454545454, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  58%|█████▊    | 288/500 [00:50<00:31,  6.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.3333333333333333, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.8571428571428571, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  58%|█████▊    | 289/500 [00:50<00:47,  4.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  58%|█████▊    | 291/500 [00:51<00:49,  4.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.8, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.4444444444444445, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  58%|█████▊    | 292/500 [00:51<00:49,  4.16it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.625, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.4, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  59%|█████▉    | 294/500 [00:52<00:55,  3.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  59%|█████▉    | 295/500 [00:52<00:59,  3.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  59%|█████▉    | 297/500 [00:52<00:44,  4.60it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.5, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  60%|█████▉    | 298/500 [00:53<00:58,  3.47it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  60%|██████    | 300/500 [00:54<01:06,  3.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.33333333333333337, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  61%|██████    | 304/500 [00:54<00:41,  4.67it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  61%|██████    | 305/500 [00:55<00:46,  4.17it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.3571428571428571, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  61%|██████▏   | 307/500 [00:55<00:46,  4.17it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.25, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  62%|██████▏   | 308/500 [00:55<00:39,  4.87it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.8, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  62%|██████▏   | 311/500 [00:56<00:43,  4.38it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  63%|██████▎   | 314/500 [00:56<00:33,  5.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  63%|██████▎   | 316/500 [00:57<00:44,  4.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  64%|██████▎   | 318/500 [00:57<00:31,  5.83it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  64%|██████▍   | 322/500 [00:58<00:20,  8.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.5714285714285715, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.5, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  65%|██████▌   | 325/500 [00:58<00:16, 10.74it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.4615384615384615, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.2857142857142857, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  65%|██████▌   | 327/500 [00:58<00:16, 10.47it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.8, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.2222222222222222, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  66%|██████▌   | 329/500 [00:58<00:18,  9.04it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.5714285714285715, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  66%|██████▋   | 332/500 [00:59<00:34,  4.82it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  67%|██████▋   | 336/500 [01:00<00:22,  7.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.5454545454545454, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  68%|██████▊   | 339/500 [01:00<00:16, 10.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  69%|██████▊   | 343/500 [01:00<00:18,  8.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.3157894736842105, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  69%|██████▉   | 345/500 [01:01<00:16,  9.58it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.4, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  69%|██████▉   | 347/500 [01:01<00:22,  6.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  70%|██████▉   | 348/500 [01:02<00:35,  4.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.8, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  70%|███████   | 350/500 [01:02<00:30,  4.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.33333333333333337, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  70%|███████   | 352/500 [01:02<00:29,  4.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  71%|███████   | 355/500 [01:03<00:21,  6.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  72%|███████▏  | 358/500 [01:03<00:17,  8.16it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.3333333333333333, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  72%|███████▏  | 360/500 [01:03<00:20,  6.92it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.8, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  72%|███████▏  | 362/500 [01:04<00:20,  6.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.5, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  73%|███████▎  | 363/500 [01:04<00:21,  6.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.6363636363636364, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  73%|███████▎  | 365/500 [01:04<00:28,  4.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.03333333333333333, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  73%|███████▎  | 367/500 [01:05<00:19,  6.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.8, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.75, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  74%|███████▍  | 370/500 [01:05<00:24,  5.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  74%|███████▍  | 372/500 [01:06<00:28,  4.57it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  75%|███████▌  | 375/500 [01:06<00:21,  5.71it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  76%|███████▌  | 379/500 [01:07<00:14,  8.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.0625, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.2, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  76%|███████▌  | 381/500 [01:07<00:12,  9.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  77%|███████▋  | 383/500 [01:07<00:12,  9.32it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.4444444444444445, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  77%|███████▋  | 385/500 [01:07<00:13,  8.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.6, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  77%|███████▋  | 386/500 [01:08<00:19,  5.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  78%|███████▊  | 389/500 [01:09<00:25,  4.40it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.5, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  78%|███████▊  | 390/500 [01:09<00:23,  4.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.4444444444444445, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.11764705882352941, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  78%|███████▊  | 392/500 [01:09<00:19,  5.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.5217391304347826, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  79%|███████▉  | 397/500 [01:10<00:12,  8.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.4444444444444445, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  80%|████████  | 401/500 [01:10<00:09, 10.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  81%|████████  | 403/500 [01:10<00:11,  8.14it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  81%|████████  | 405/500 [01:11<00:13,  6.84it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.18181818181818182, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  81%|████████▏ | 407/500 [01:11<00:15,  6.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.3333333333333333, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.5, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  82%|████████▏ | 409/500 [01:11<00:15,  5.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.8, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  82%|████████▏ | 411/500 [01:12<00:13,  6.49it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.3636363636363636, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  83%|████████▎ | 413/500 [01:12<00:15,  5.72it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  83%|████████▎ | 415/500 [01:12<00:10,  8.09it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  83%|████████▎ | 417/500 [01:13<00:11,  7.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.5, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.4, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  84%|████████▍ | 420/500 [01:13<00:11,  6.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.3, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  85%|████████▍ | 424/500 [01:14<00:10,  6.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.2857142857142857, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  85%|████████▌ | 425/500 [01:14<00:17,  4.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.6, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  85%|████████▌ | 426/500 [01:14<00:17,  4.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.5, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.5, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  86%|████████▋ | 432/500 [01:15<00:08,  8.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.8571428571428571, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.8, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.19999999999999998, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  87%|████████▋ | 435/500 [01:15<00:06, 10.47it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.4444444444444445, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  87%|████████▋ | 437/500 [01:16<00:09,  6.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  88%|████████▊ | 439/500 [01:16<00:08,  6.92it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.8, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  88%|████████▊ | 441/500 [01:17<00:10,  5.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  89%|████████▉ | 444/500 [01:17<00:09,  5.72it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.8235294117647058, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  89%|████████▉ | 447/500 [01:17<00:06,  7.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.4444444444444445, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.33333333333333337, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  90%|████████▉ | 449/500 [01:18<00:07,  6.96it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.7499999999999999, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  90%|█████████ | 451/500 [01:18<00:06,  7.09it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  90%|█████████ | 452/500 [01:18<00:08,  5.48it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.6153846153846153, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  91%|█████████ | 456/500 [01:19<00:06,  6.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.5, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  92%|█████████▏| 458/500 [01:19<00:05,  7.14it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  92%|█████████▏| 460/500 [01:20<00:07,  5.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  93%|█████████▎| 463/500 [01:20<00:04,  8.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  93%|█████████▎| 465/500 [01:20<00:04,  7.48it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  93%|█████████▎| 467/500 [01:21<00:05,  5.57it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.08333333333333333, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  94%|█████████▎| 468/500 [01:21<00:05,  5.71it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  94%|█████████▍| 469/500 [01:21<00:06,  4.71it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.47619047619047616, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  94%|█████████▍| 471/500 [01:21<00:05,  5.71it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.4444444444444445, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  95%|█████████▌| 475/500 [01:22<00:02,  8.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.4, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  95%|█████████▌| 477/500 [01:22<00:02,  8.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.5, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  96%|█████████▌| 479/500 [01:22<00:02,  7.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.2857142857142857, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.3846153846153846, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  97%|█████████▋| 483/500 [01:23<00:03,  5.60it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.8, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.3333333333333333, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  98%|█████████▊| 488/500 [01:24<00:02,  5.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.33333333333333337, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.046511627906976744, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.375, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  98%|█████████▊| 491/500 [01:25<00:01,  7.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.5714285714285715, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  99%|█████████▉| 494/500 [01:25<00:00,  9.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.33333333333333337, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  99%|█████████▉| 496/500 [01:25<00:00,  6.63it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.1, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  99%|█████████▉| 497/500 [01:27<00:01,  2.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow: 100%|█████████▉| 499/500 [01:28<00:00,  2.09it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.8, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow: 100%|██████████| 500/500 [01:40<00:00,  4.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344294b110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442e248d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a6f2040>, 9883885.931112433)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344294ad50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a3e3230>, 9883885.970452735)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442fa0050>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534428d61d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a8007c0>, 9883889.416357808)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534428d64d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534429640d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442873250>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442805550>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534428b9190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a8022e0>, 9883888.596422192)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534428b9850>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a4dcad0>, 9883888.63398444)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534428737d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a4cb540>, 9883888.13675181)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442964210>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a4cadd0>, 9883888.202582253)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442853190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344283a050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a4def20>, 9883888.339665469)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534428a4750>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534427aa910>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a6f2c10>, 9883890.924600303)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153443009510>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344277b3d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a930520>, 9883890.595105015)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344277b4d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534427f7d10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a932270>, 9883891.96354941)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344266b790>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534427ce6d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534427f4490>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a9312b0>, 9883891.024695864)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534427cee50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a803230>, 9883890.77079462)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534427f4bd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344283b1d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a9da120>, 9883897.090303743)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534426d5110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442cd1810>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458536430>, 9883895.408383045)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534425567d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442443c50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153443ed6350>, 9883895.874496719)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442443b10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344240b5d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a932f20>, 9883897.746072764)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344240aa10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442488350>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153443ed5b00>, 9883896.117162935)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442488210>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534425b1190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153443ed6660>, 9883896.688541628)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442496e50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344235cc90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153443ed69e0>, 9883898.509122252)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344235cd90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534425f7a90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534423c5630>, 9883899.602686567)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344224ced0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442227d90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534423c7cb0>, 9883901.889397925)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442233a10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534426f0fd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534423e7810>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442203110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442225650>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442234d10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153443ed56a0>, 9883898.921166124)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442414550>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534423c5b70>, 9883899.870721668)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442225510>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534424c6d60>, 9883899.634931251)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442202f10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534423c4d70>, 9883898.276052348)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534423e76d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534423c63c0>, 9883898.968617285)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442327650>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344212e890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534422c06e0>, 9883912.8638428)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344212eb50>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442389990>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153443ed7000>, 9883901.440958492)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442402b90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534422e8050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a931550>, 9883901.915238945)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534422e8a50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534421f4310>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534421a1860>, 9883904.329962905)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534421f4650>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344215a2d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442180110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442197d90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534421b3a50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534422c38c0>, 9883902.012672607)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442180750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534421a0910>, 9883901.9282421)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534421b3f10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534422c1010>, 9883902.4499623)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344215a0d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534421a0360>, 9883902.123560973)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442197c90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534421de990>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15344208d320>, 9883905.861468168)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534421de350>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534420db8d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15344208d160>, 9883905.578257432)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442153810>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441f35b10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15344208dcc0>, 9883905.071020996)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441f35d90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441f888d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441f8c210>, 9883905.952274565)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441f88790>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534421c4d90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344202ab50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344207fa10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344200e590>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534420ec850>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441eff690>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534421a02f0>, 9883904.488818176)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344207f8d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534421a35b0>, 9883904.510759892)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534420ec650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441f8cbb0>, 9883910.980515113)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441f90b90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534421a3bd0>, 9883903.69387728)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344202abd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15344208c4b0>, 9883904.123836452)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344200ea90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15344208e120>, 9883904.931088174)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441eff550>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441e16690>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441f8eb30>, 9883908.604475709)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441e16910>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441e76450>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441e6de10>, 9883917.589502648)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441e767d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441e8d990>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441e6cad0>, 9883909.283098249)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441f9a990>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441f6ed90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441fe4790>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441fe70d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534421a1a90>, 9883906.909784455)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441fe6f90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15344208c130>, 9883907.259204764)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441f6e010>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441f8dbe0>, 9883906.84543656)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441fdee50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441e66290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441e6f000>, 9883909.543251475)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441e66990>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534420ab390>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441d70de0>, 9883911.279755253)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534420aa850>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441ddee10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441d722e0>, 9883911.567754276)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441ddf190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441d4a950>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441d5f9d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441d700c0>, 9883909.87121948)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441d5f890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441e6f850>, 9883909.692262916)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441d4a750>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441ebf790>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441c5c980>, 9883912.207253795)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441ebfe10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441c40610>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441d731c0>, 9883912.212072683)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441c403d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534420b8c10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441c5f7e0>, 9883914.600844437)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441b1bfd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441c92050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441c5cf30>, 9883912.122924348)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441c92710>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441c832d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441b54830>, 9883915.024880659)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442d22510>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441b0db50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441b56e40>, 9883916.35554745)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441c79fd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441ad9ed0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441a46970>, 9883919.180772856)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441ada250>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441aeec90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441a470e0>, 9883920.27687554)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441aeeb50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441a2cc90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441a2e290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441e6e890>, 9883915.563788336)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441a2ca90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441c5f700>, 9883916.01669242)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441a2e250>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344194b8d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441938910>, 9883918.422790164)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441af4e50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534419818d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534419a1fd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441939d30>, 9883920.404015029)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441ade350>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15344193a0b0>, 9883919.148034379)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534419a1e90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534419d6550>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534419e9690>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15344193aba0>, 9883919.916998062)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534419d6c10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534419e9390>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15344193b2a0>, 9883920.46236618)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534419eb810>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441930ed0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441a475b0>, 9883919.870492524)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441930c90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441803010>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534417ffc50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441d726d0>, 9883919.8876612)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441802fd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441828b40>, 9883921.536135988)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441800c10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441914b90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441828d00>, 9883919.461309116)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344181fa90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344204c450>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441885e10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441bbb790>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441a4a990>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15344193b0e0>, 9883919.479152624)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534419e9510>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441a45710>, 9883918.419996843)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441a4a910>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441829400>, 9883922.263957117)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344204c110>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15344193bcb0>, 9883920.205774125)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441885c50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441938360>, 9883917.634719877)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441bbb750>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441891690>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442de5c90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442990290>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442968650>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534429d5810>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442cbfdd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442c34550>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344281da10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344281f450>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344283a1d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442815ed0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534429aa790>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534426fd690>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534428a7b10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442715d50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344272f890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344281f990>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442890dd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442790690>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534427a2f90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442888190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442cbc210>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442642ed0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442610450>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534426683d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344266e790>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534426b5910>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534426cd890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534427f5250>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344262e910>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534426dcb10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442501f50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442528910>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344252bf90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534425543d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442569450>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442584b90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534425bf4d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534425eb310>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534424024d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442408810>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344242e1d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442415490>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442441a10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442495210>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442529790>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534424e30d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442305c10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442313210>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442334550>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534427a3110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442354750>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442389a10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344237de50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534423a2b90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534423c8e50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442262d10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534424c0590>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534422b6690>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534422cf350>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534422f34d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442279d10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442105e50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344212c950>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442133510>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534421dc150>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442c12110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534420405d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344201f390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534421c5090>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534420a6950>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442058b90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442000fd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442041890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441f72ed0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441f99890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441e14690>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441fc7190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441dfce50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441ea2d50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441ebfdd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534420aba90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441e253d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441e8bdd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441d115d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441d18a50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441d3a510>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441d88250>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441d12190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441c06190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441d9c390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441d4bc10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441c52610>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441d750d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441c353d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441f4a350>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441cc4090>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441cece90>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441c7a050>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441b19b10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441c93210>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441f19110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441b872d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441b95010>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441baf090>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441cd1410>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441b7b990>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441bbabd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441a09010>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441a32790>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441a584d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441a88c10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441a48fd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441aa0a90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441ab9d10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441ac2410>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441914450>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441926f90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534419cddd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441d72970>, 9883918.4250287)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441903a50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15344182a3c0>, 9883921.287356853)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441891150>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a3e3000>, 9883889.157252364)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442de6090>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a6f22e0>, 9883886.783054136)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442990610>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a3b4e50>, 9883886.535741301)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442968a10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a4deb30>, 9883886.860900346)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442d30e50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a4df2a0>, 9883889.60707034)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442c96d10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a4df460>, 9883889.604118744)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442c348d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a4dd940>, 9883887.791853052)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344281e0d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a4dee40>, 9883888.016023956)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344281f650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a4dfd20>, 9883889.133072944)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442839fd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a3b5f60>, 9883889.404511308)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442815d90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a3e18d0>, 9883892.046643632)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442891ad0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a801080>, 9883893.187756348)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534426fd550>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a3b60b0>, 9883892.483204233)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534428a7c50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a800a60>, 9883889.520995775)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442715c10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a802d60>, 9883890.109081808)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344272f550>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a3e0670>, 9883890.312135687)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344281f810>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a803a80>, 9883890.409140574)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442c4df50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a8001a0>, 9883890.579693602)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442e08cd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a6f10f0>, 9883893.092583332)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534427a2e50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a932430>, 9883891.478955915)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442888290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a3e2ac0>, 9883891.67514746)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442cbf210>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a932a50>, 9883893.625840936)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442642990>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a931470>, 9883906.72016828)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534426102d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a9339a0>, 9883892.428767228)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534426688d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a9331c0>, 9883894.059425509)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344266e610>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a933690>, 9883892.825772917)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534426b5550>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a8008a0>, 9883893.00371239)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442888410>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1533fadeee40>, 9883893.400223482)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534427f5390>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a932d60>, 9883893.360136716)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344262e710>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1533fadecfa0>, 9883895.698072976)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534426dd150>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1533fadef700>, 9883893.071299069)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442502590>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a4ddb00>, 9883895.391948616)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442528dd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1533fadecd70>, 9883894.723449802)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344252be90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a9d8520>, 9883894.419568907)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442554910>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a4df620>, 9883894.1501343)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442569b10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a6f39a0>, 9883895.305820595)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344257ee50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458eb2b30>, 9883895.62293063)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534426b6e50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458537310>, 9883894.494893307)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534425eb110>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1533fadec440>, 9883896.41563792)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534424022d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a800de0>, 9883896.201503461)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344240b310>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a3b49f0>, 9883896.279695123)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344242e910>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153443ed4830>, 9883895.022111287)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442415790>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a802eb0>, 9883896.242647031)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442441d90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1533faded400>, 9883896.978526803)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442495010>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a930670>, 9883899.642717581)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442541e10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534424c4910>, 9883897.140264949)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534424e2d10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534424c5240>, 9883897.832047824)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534423058d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534424c55c0>, 9883897.740126751)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442313010>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534424c5e10>, 9883899.831242047)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442334b10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153443ed6200>, 9883897.961404445)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534427a3910>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534424c67b0>, 9883908.053112527)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442354e10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534424c74d0>, 9883899.463008257)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442389810>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1533faded860>, 9883898.370889343)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344237dd10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534424c7cb0>, 9883900.604330797)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534423a2a50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534423c4520>, 9883900.600306744)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534423c8c50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1533fadee120>, 9883899.885197448)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442262bd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534423c7930>, 9883901.810899835)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344231d210>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534422c0360>, 9883900.280276733)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534422b6f50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534422c0d00>, 9883901.291507104)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344241f990>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534422c14e0>, 9883911.856177345)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534422f11d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534423c7700>, 9883901.348024981)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442279950>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534422c18d0>, 9883901.777620515)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442226350>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534422c23c0>, 9883903.172431726)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344212cdd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534421a0750>, 9883902.550783675)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442203e90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534421a0f30>, 9883903.995466435)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534421dc810>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534421a1630>, 9883902.649759362)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442324950>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534422c34d0>, 9883904.029241616)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442040cd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534421a3150>, 9883906.345685195)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344201ee50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a6f1630>, 9883904.165403288)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442227b50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15344208c6e0>, 9883904.900923235)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534420a5210>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15344208c980>, 9883905.066157442)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344200f690>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15344208de80>, 9883905.162749056)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534420eee50>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15344208fb60>, 9883906.620680591)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441f417d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534421a26d0>, 9883906.202526487)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344224ee50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441f8cd70>, 9883906.229396252)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441f99750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534421a1a20>, 9883908.74590274)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441e14490>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15344208c1a0>, 9883908.941679467)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441fc4550>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441f8f4d0>, 9883907.409972636)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441dfca90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441e6cf30>, 9883909.331256228)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441ea2c10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15344208c280>, 9883910.398998236)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441ebee50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441f8f8c0>, 9883907.921830295)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534420ab4d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441f8e890>, 9883908.90527626)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441e24650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441f8d010>, 9883907.971317256)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441e8bc90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534421a2740>, 9883909.217073934)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441d11b50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441e6eac0>, 9883909.794838)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441d19110>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1533fadee190>, 9883913.84762242)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441d3b190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441d707c0>, 9883910.618893648)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441f18fd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441f8ef90>, 9883911.952403162)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441d8a650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441f8edd0>, 9883911.32933778)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441e744d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441d72f20>, 9883914.056197846)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441d9d350>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441d71a90>, 9883912.202234907)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441d4b350>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441c5c050>, 9883912.35788785)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441c52b10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441d70520>, 9883910.855996264)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441d74690>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441d71550>, 9883912.945136145)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441e66e50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15344208f620>, 9883914.88142088)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441e74190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441c5db00>, 9883914.949148096)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441d4a9d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441c5e820>, 9883913.231858833)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441ced650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441c5da90>, 9883913.2002001)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441c79f90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441c5f230>, 9883914.231606876)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441b19910>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441d72740>, 9883916.788564311)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441ddee50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534424c71c0>, 9883913.829056213)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441dde310>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441b550f0>, 9883916.749420308)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441b87190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441b558d0>, 9883917.903797334)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441b95250>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441e6d1d0>, 9883915.326308165)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441baee90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441c5c910>, 9883913.768761093)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441cd1710>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441c5f4d0>, 9883913.529390195)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441b7b850>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441b55cc0>, 9883915.355722671)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441b65c90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441b55390>, 9883917.394777644)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441a08e90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441a44050>, 9883916.789899724)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441a33190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441b573f0>, 9883919.207797304)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441a5bb50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441a452b0>, 9883916.470943488)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441a88a10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441a45400>, 9883917.223519528)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441a49190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441a45a20>, 9883916.324932076)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441bd8fd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441d72f90>, 9883917.388960436)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441aba450>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441a46040>, 9883917.398995483)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441ac2a50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441b55f60>, 9883918.421553265)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441926d90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15344193a970>, 9883919.13986518)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441e662d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534426cd2d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a3b41a0>, 9883894.117641062)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534426cc4d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534417191d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441755290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458eb3b60>, 9883921.87778964)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344171a2d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441829010>, 9883922.416413592)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441755690>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344171bb10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441d71cc0>, 9883927.392194195)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441719350>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534419cd950>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441802790>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534421a3540>, 9883923.82861505)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441803d90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441ebdcd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441a46d60>, 9883922.573923944)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441ebc490>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441fd6190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534420a74d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534424c5d30>, 9883922.435912896)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441fd6050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534424c4e50>, 9883922.828865612)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534420a6c50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442292e90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a932970>, 9883923.844094254)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345a391f10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344177ed50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441b5b910>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441829a20>, 9883922.761600537)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441afcf10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a802580>, 9883924.364433696)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534424c9c50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534417fc490>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534425e98d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a2b0210>, 9883923.981856938)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534417fda50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a2b3620>, 9883924.274695957)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534425e9210>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534427e9d90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344289f190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534418f0850>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534418f39d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534417038d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534418608d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344172db50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441a47690>, 9883921.850909386)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534417310d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15344193a740>, 9883921.464950109)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441703790>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15344208dbe0>, 9883924.564973194)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534427e9ed0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a2b35b0>, 9883924.11547074)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442510490>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458436190>, 9883920.596153112)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534418dfed0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153443f19470>, 9883922.164917754)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534418f37d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441a46900>, 9883921.278595047)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441adafd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441e6e9e0>, 9883921.258425154)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344172dd90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534429830d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442992590>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a0926d0>, 9883924.870137371)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442980f50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534419394e0>, 9883924.92028476)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442992410>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442c49110>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a0917f0>, 9883927.581059432)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442678b10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344177ebd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534423c6350>, 9883924.507570973)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344177db90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442c86a90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442cc8190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a091390>, 9883926.127494777)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344177c750>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a9da0b0>, 9883925.854959035)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344266f490>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441c0d790>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442c35150>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a2b3f50>, 9883926.058201663)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441ebd590>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a093000>, 9883925.754521215)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442c35810>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344177ffd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a092190>, 9883925.69046081)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442c7e550>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442df0e50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15346030b000>, 9883926.17587415)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441730a90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442d47c10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442d6cd10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442769990>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344291e110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442911f90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a2b2f90>, 9883924.051255656)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344291cc10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a2b0de0>, 9883924.179935368)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442912310>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534422c3150>, 9883927.617242329)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442d47690>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1533fae18c20>, 9883927.335352996)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442d6f310>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a2b3850>, 9883923.785104744)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534428b8910>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534419eb990>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1533fae18360>, 9883927.20851515)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442e73810>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442c035d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1533fae19470>, 9883926.513745012)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442e82090>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442fed510>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442e09190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441a47af0>, 9883928.698175082)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442fef110>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1533fae19240>, 9883927.790184774)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442e08d50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442c02a10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442f4f790>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1533fae183d0>, 9883927.081653493)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442c002d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345ad5e580>, 9883928.683150783)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442c1e890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534430cd3d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345ad5d390>, 9883930.054780422)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534430cee50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441ea0a50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153443029250>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345ad5f770>, 9883928.95637284)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153459f20750>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344309d010>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345ad5f460>, 9883929.984544188)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344172e090>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153443060310>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153443076bd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345ad5f1c0>, 9883929.3093916)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153443063bd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345ad5f3f0>, 9883929.568896111)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534430773d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442d540d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442e24850>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442f45f10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15346030b310>, 9883927.734856987)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153443028c10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1533fae1b230>, 9883927.404821636)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442f46b90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534421f7950>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a090830>, 9883927.63802024)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442d54290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441c5ef20>, 9883927.14972622)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442e26a50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1533fae181a0>, 9883927.537316717)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534420e1d10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442d6c750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1533fae1a580>, 9883930.26564342)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534431a4fd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534430540d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a5f3af0>, 9883933.7057908)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153443057210>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534431ea6d0>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344300b4d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441a46e40>, 9883930.710248793)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534431e9650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a5f1860>, 9883937.12769124)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344300bed0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153443046150>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344315bd50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a5f3930>, 9883930.115088608)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441ebdb10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a5f1a20>, 9883932.942928256)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153443158810>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442ed9b90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345bc8b1d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a5f0130>, 9883931.625931447)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442ed9650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a5f0c90>, 9883930.5828959)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442d544d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345bc6a4d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441829c50>, 9883931.840931242)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345bc68290>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442e09b90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a5f1010>, 9883930.86907305)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442e09790>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534429d6ad0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345bc15390>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a0919b0>, 9883931.277731547)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534429d4e10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345bbec150>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a5f25f0>, 9883932.307379665)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345bbee990>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345bc4a310>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534417021d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442e9d290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345983c520>, 9883932.27028988)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345bc16350>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153443f1b150>, 9883932.217457816)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345bc1f410>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534431b9410>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a092dd0>, 9883929.914272947)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441a16e50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a5f26d0>, 9883928.939166635)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442e9df90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a8db230>, 9883929.619934924)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534431ba010>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345d3f8650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534580f1550>, 9883932.611427529)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345d3f8890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345ad5f700>, 9883939.21330658)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534421b1590>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345ad5e190>, 9883939.506614763)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344236af50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442e7afd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442e30190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345bc56510>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442e39c10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345d407710>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345d3edcd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a093a10>, 9883933.94006732)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442e79410>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534580f0bb0>, 9883932.94653192)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345bc89bd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a5f0830>, 9883932.03035553)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344307c590>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534580f31c0>, 9883931.818830244)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344282a710>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534580f0910>, 9883932.806806272)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345d406e50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534580f3cb0>, 9883932.935171587)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345d3ef010>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534427ab710>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534428ea910>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534428d4dd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534427a3a50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442912910>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442c12190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442d3c950>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344308dc50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345ad5cad0>, 9883939.726086313)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344308cc90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442f35210>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345bc54750>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344310ab50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345d360850>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345bc08310>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345d3eefd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345d3f8690>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345bbc4090>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345d3adc10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a5f3a10>, 9883940.787249472)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534431f5790>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a5f0fa0>, 9883939.338437883)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442ddf4d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a5f0ec0>, 9883940.257425599)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345d3ad690>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345ad5f070>, 9883940.590057304)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534427c0e90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345ad5d470>, 9883941.582852768)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534428eb4d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345ad5d7f0>, 9883940.463225933)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534427e1cd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345ad5cfa0>, 9883939.540674377)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534428adfd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345ad5eb30>, 9883943.772755316)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442911b50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345ad5e660>, 9883939.15746277)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442c13110>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345ad5da20>, 9883942.630385676)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442d487d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a5f3000>, 9883941.296446761)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442f52c10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a5f2d60>, 9883940.015457656)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345bc6c090>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a5f0440>, 9883939.577560766)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153443109590>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a5f1a90>, 9883940.17582862)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345d3626d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a5f2040>, 9883940.240964368)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345bc08c50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a5f04b0>, 9883939.35561997)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345d3ec990>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345d546690>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a5f3460>, 9883939.754609656)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345d544650>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345d577990>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345d532d10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458d9dcc0>, 9883941.730059212)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345d533850>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345d70f290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a5f2dd0>, 9883945.005769625)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345d70f790>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345d66b490>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458d9f2a0>, 9883941.472655972)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345d66b190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345d537c10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345d56f950>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534421a3070>, 9883943.64202027)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442f1e690>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458d9fe00>, 9883941.549353454)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345d56c250>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345d52b150>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458d9c4b0>, 9883943.155488074)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345d672b50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345d71bf50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345a833410>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458d9c750>, 9883942.184746588)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345a813910>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458d9cec0>, 9883942.611412045)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345a8329d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345d553310>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458d9c360>, 9883942.25038132)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345d553250>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345f66e2d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345bc95710>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458d9e890>, 9883942.802108904)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345d69ee50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459b5aac0>, 9883942.671017846)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345bc96e50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345d5ac150>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534431426d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459b5a270>, 9883942.688098269)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345bbe4ed0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459b581a0>, 9883944.294072067)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153443142750>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345d3b4050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15344208ce50>, 9883941.98420011)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345d574750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459b5bee0>, 9883946.33515086)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345d54b190>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345bbf7b50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345ad5c280>, 9883943.290820396)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345bbf7dd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345d56e850>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a2b3b60>, 9883944.36643868)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345d561210>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442888750>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1533fafcd650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459b58980>, 9883946.407315912)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442864690>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345d70b090>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153443119f90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459302200>, 9883947.138441848)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345d708b10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a4df770>, 9883944.676777748)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345d5a7c90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345a853110>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458d9e660>, 9883944.919204412)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345a853190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345999d3d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345a7d8dd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441e6cfa0>, 9883949.352779685)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345a52f690>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a6f14e0>, 9883946.392502952)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345a7dbf90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345ad75ad0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a6f3620>, 9883945.41654038)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345ad765d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345a544290>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345baee310>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a6f2f20>, 9883946.55214004)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345a5478d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a6f3a10>, 9883946.956897832)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345baee410>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345a4f40d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1533fad7c890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a6f0910>, 9883945.821123788)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345a6b0910>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345a2c11d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345d4d5b50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534580f1710>, 9883943.978332479)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1533fafcf050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a3b6f90>, 9883945.691513067)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1533fad7d610>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a6f25f0>, 9883946.039565623)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345a2c3590>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a3b52b0>, 9883946.192655258)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345d4d6e50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345a68a650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a3b6f20>, 9883951.703863604)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345a688c10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345a796390>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a3b7620>, 9883946.536760552)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1533fadd9c10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345d51b510>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a3e3af0>, 9883947.034550205)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345d51b150>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345a029850>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345a0cbf90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a3e0980>, 9883948.072961483)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345a02b250>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a3b4d70>, 9883952.188687684)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345a0c80d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345a2b9950>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a3b4590>, 9883949.92401254)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345a28fe10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345f586890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a3e1f60>, 9883949.181071328)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345d3245d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15358dcee910>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a4c87c0>, 9883949.858458018)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345d4cf610>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345a044850>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a3e2510>, 9883948.650581604)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345a045f50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153459b74950>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1533fadbfb50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345a2f2150>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345d4b2ad0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345a577790>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459302dd0>, 9883946.371468425)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345a577e90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a4ca970>, 9883950.2423014)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153459b748d0>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a4c8050>, 9883949.34075032)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1533fadbe710>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a3b6c80>, 9883945.991568932)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345a2f0fd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a3b7e00>, 9883946.96429768)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345d4b2e10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345d4b40d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a4c9550>, 9883951.715397857)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345d4b5310>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534599b8490>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345d503e50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a4c9cc0>, 9883950.175997697)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534599bb710>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153459c4b550>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459e755c0>, 9883956.284007777)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534596dc990>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534594f7350>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459e75b00>, 9883950.559878226)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153459370210>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345f650e90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15344193a200>, 9883949.171682112)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345f652250>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153443e37950>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459e77620>, 9883950.026609372)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153443e36c90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153443fcb6d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459e77930>, 9883952.913618311)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153443fcb190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153443f5ec10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459a397f0>, 9883955.919591196)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153443f5e390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345d483f50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a4cb150>, 9883951.087187476)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345d483d90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1533fad46990>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534581c9f50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153459f42810>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345d51a9d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459e75710>, 9883951.252115125)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345a083450>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345d480850>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534429a42d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459e76c10>, 9883949.9288995)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345ad4cb90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a19bc40>, 9883951.378561104)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345a460fd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a19a190>, 9883953.246678198)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534581cae50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15344208c210>, 9883948.575514784)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345d33cc10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458d9f3f0>, 9883948.465374785)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345d518f90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459e77700>, 9883949.246445196)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345d4834d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458184d10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459e76900>, 9883953.682426602)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534581871d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458295050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a19ac80>, 9883952.622911526)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458294d50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458249f90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a199780>, 9883953.048702858)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345824bad0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458248150>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534583c3a90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a19b700>, 9883954.036645617)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458248490>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a4cb380>, 9883953.67373478)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534583c3e50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345838f210>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459301a90>, 9883958.881206572)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345838ee50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345834d050>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458354ad0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459c5c280>, 9883953.964033436)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458356f90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534582fd990>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459c5f460>, 9883954.876491388)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534582fd590>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1533fad37f10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a1990f0>, 9883955.906981751)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534582aaa90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345d50ddd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153443f8e850>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345829de10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345826d590>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458d9f540>, 9883953.88921646)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345adccc10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534580e9050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459e74830>, 9883951.912095847)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345d50c910>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459e75470>, 9883951.984075136)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534580eb7d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534424c40c0>, 9883952.11722662)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345826f590>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459c5e200>, 9883955.90816192)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345d46d550>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a1996a0>, 9883952.34131142)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153443f8c690>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a19a3c0>, 9883951.829181155)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153459f601d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345a450250>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458430e90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459c5def0>, 9883954.65985559)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345d332450>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534583e6e90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a4cba10>, 9883955.838705381)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534583e5590>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534585bd250>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459a390f0>, 9883960.807299657)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534585bf790>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458593990>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458532650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459a39630>, 9883956.44816594)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458593e10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459a3b150>, 9883956.089721693)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458531810>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458514c50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459a3b700>, 9883956.206677085)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458515910>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534584eca50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459c5ef20>, 9883956.900625024)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534584ee150>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534583b7910>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534586584d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459a383d0>, 9883956.835890535)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534585b10d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459c5d240>, 9883957.878971403)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458659f50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345834ea10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458633b50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459a3a5f0>, 9883956.953261856)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345834e190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459a39c50>, 9883957.232445039)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345865a1d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345849a090>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345993ee40>, 9883957.144291632)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345879b490>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534584c6150>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a199010>, 9883955.9544638)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345a907950>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345824acd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a19a7b0>, 9883954.89030812)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534584c6950>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459e76ba0>, 9883956.682391424)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534584828d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459a38e50>, 9883954.940237613)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345824b250>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345857cad0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459c5f850>, 9883957.974982653)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345857f110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534587bb7d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345993e580>, 9883957.710046548)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534587bbb10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458741cd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345993c130>, 9883958.456029644)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458743190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458702f10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345866b590>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345993c980>, 9883958.939239373)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534587013d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345993e7b0>, 9883958.020840136)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534585159d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534588ba1d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345887a0d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459e75a90>, 9883958.962917585)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534588b9950>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345993d320>, 9883959.204353089)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458878790>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534587e8750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534595130e0>, 9883959.2646127)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534587ea050>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458800b10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459513070>, 9883958.254897932)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458800750>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534587e72d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534589a4e90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459512970>, 9883960.018512549)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534584307d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345993d0f0>, 9883959.403709153)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534589a5490>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458951290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534595115c0>, 9883959.175360484)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458963190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458978350>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458927e90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459a3a350>, 9883959.923973465)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458978110>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534595110f0>, 9883959.010799723)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458924c50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345867d210>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534595112b0>, 9883959.787194544)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345867c690>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458903390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458a96d50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459511e80>, 9883965.682379818)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534585a1a90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458a45150>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458a02e50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458879a90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345993e900>, 9883962.281557793)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458742c90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458b855d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345973b000>, 9883967.321964089)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458b93190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458b50cd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459513770>, 9883962.944008924)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458b518d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534587039d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458b0d390>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459739a20>, 9883961.637930874)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458b2ee50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345973a430>, 9883961.929099204)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458b0ddd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458ba0c50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345973a190>, 9883961.986300232)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458ba2590>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343621b690>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458c7d450>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345973a900>, 9883961.882023724)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343621bc10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458c3ddd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345993c3d0>, 9883960.305144656)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458a97090>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345993dd30>, 9883960.826974163)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458a45490>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459a3b7e0>, 9883962.138739975)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458803390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534589fc110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458a6c950>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459c5ce50>, 9883959.730150767)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534589cee50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a198130>, 9883962.66552722)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153443f5ec50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345973a9e0>, 9883961.226017075)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458c3cbd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345973a7b0>, 9883959.73591866)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534589fcd10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458533710>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345962c3d0>, 9883963.066160023)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534586f8090>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458a9e450>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345962c750>, 9883962.06872782)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458615110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345890bc90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458d92d90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a3b7070>, 9883962.765978888)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534589091d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345962dcc0>, 9883964.97228872)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458d90690>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458d51290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345962cad0>, 9883962.94880102)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458d52910>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458c3e750>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458d23d90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345962c050>, 9883962.996922052)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458c3fa90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459512890>, 9883964.121612856)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458d21950>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458b0a850>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459511ef0>, 9883963.346184475)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458ed2910>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458e5add0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458dfa650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345962e970>, 9883964.956795152)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458e5b310>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459f8ec80>, 9883964.156334529)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534596a0950>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345841c190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458e26650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459f8d390>, 9883965.409815587)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458e90650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459f8e9e0>, 9883969.515008297)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458e27890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458cfd090>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458cd9dd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1533fad26f90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458be4190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345962e820>, 9883962.159849908)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458be5850>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345962f7e0>, 9883965.257777095)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458e132d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459739b70>, 9883965.454886813)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458cd9d50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345962fee0>, 9883961.992263045)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1533fad25f50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458d22150>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458f77210>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345962cd00>, 9883966.783879302)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458f77e50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458f07a90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459f8fd90>, 9883968.236787843)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458f07b10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534596c60d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459f8c7c0>, 9883966.989534102)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458d48310>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458e4c390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153459090b50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459f8c520>, 9883966.449561767)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458e4ea50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345907ad10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459d755c0>, 9883966.204613188)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345907a890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345901d610>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345973af90>, 9883966.227514286)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153459016e50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458f3d8d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459d74830>, 9883968.227766164)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534588a8a90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458e91490>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345841f2d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459d74280>, 9883968.31852427)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458e92750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345962c830>, 9883967.124937965)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345841f110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153459316410>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459a38520>, 9883966.5262404)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458d20c50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458e02e90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458efd490>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459f8cde0>, 9883965.929480076)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153459090f90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458a00450>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345993d390>, 9883965.352436194)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458f28950>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345bae2270>, 9883967.361534417)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153459317250>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345bae0130>, 9883965.245767027)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458e03790>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459f8c910>, 9883965.483367492)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534590c3450>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153459532150>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459d76f90>, 9883969.706774509)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458d1a590>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344174a6d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345863c050>, 9883970.658920964)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534417486d0>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344176e090>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345863fee0>, 9883969.544043606)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344176f690>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441177c10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344118fdd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345863cec0>, 9883969.234613348)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441177ed0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459d75f60>, 9883969.562739024)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344118fc90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534411a3250>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345863c600>, 9883972.25698883)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534411a3990>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458efd210>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345863eb30>, 9883969.373698913)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458efcf50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534411b6a10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345863de80>, 9883969.596452508)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534411b66d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153459127010>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534411e6690>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459738ad0>, 9883969.625677828)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153459063b90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458435320>, 9883970.55831755)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534411d9f10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534410011d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345bae29e0>, 9883970.396824004)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441000fd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441002fd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153459170590>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345925d0d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458fe9190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441175690>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345863c210>, 9883967.545591015)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345911d6d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345863ee40>, 9883968.311756631)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441175550>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459d748a0>, 9883970.415365512)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441002d10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459d77460>, 9883968.331080856)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458c11c10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345863cbb0>, 9883968.527350513)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534590a7450>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441026f10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458434210>, 9883978.999510922)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441026dd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458e89c50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441065910>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458435860>, 9883970.63484718)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458f07850>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458434830>, 9883975.6147797)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441066010>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344100ba90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534410985d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458747d20>, 9883972.041762142)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441067350>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459738590>, 9883970.9431848)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344108ee50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534410c6910>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345962c590>, 9883976.657539807)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534410c6fd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534410877d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534410d98d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345863fd90>, 9883977.160808496)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153459752b10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458745860>, 9883972.40095236)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534410d96d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344109dd90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534410e7710>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458747cb0>, 9883971.597762741)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344109e3d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458745390>, 9883971.88518926)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534410e75d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440f06950>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458747a80>, 9883971.82393464)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440f06790>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440f15710>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458746e40>, 9883972.427755332)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440f15e50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344109ec50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441043dd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441055c90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459f8c280>, 9883972.599706644)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344109ecd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534584348a0>, 9883970.757152263)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441043c90>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458435c50>, 9883970.439346172)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441054750>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440f4fa90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534587442f0>, 9883973.180716025)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440f4f950>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440f676d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458a737e0>, 9883973.165609691)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440f64f10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440f30490>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458a73000>, 9883974.25756928)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440f305d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440f97190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440faa5d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458a71e80>, 9883974.138853475)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345893f990>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440fb4110>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345863dfd0>, 9883973.839963716)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440fb4610>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534410f0490>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345962def0>, 9883978.097612187)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440f28c10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344101ca50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440e12110>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458861010>, 9883974.015260737)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344101cad0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534588625f0>, 9883976.128556252)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440e11fd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440e21890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458861da0>, 9883975.158308333)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440e21ed0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440e3ca50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458860830>, 9883975.78331012)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440e32810>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458f2c750>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440e56bd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458861630>, 9883975.788027927)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440fb4ed0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458861e80>, 9883976.691473372)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440e56a90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440e08f90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440e76550>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440f8f5d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458435470>, 9883973.496360255)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440faaad0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458a70910>, 9883972.854389528)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440f8cc10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458862040>, 9883974.967634434)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440e09590>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458436e40>, 9883975.926636497)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440e76e50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534411755d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458a71550>, 9883972.459888373)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441175ad0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440e96450>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458a70bb0>, 9883976.423634307)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440e966d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440feba10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458863a10>, 9883976.775895506)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440ec1350>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440ed6590>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440ee5f10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458a73ee0>, 9883978.098887447)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440ed4750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458746200>, 9883976.568224523)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440ee5dd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440ec0390>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345896d400>, 9883976.165749691)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440ec0890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440d2fd10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345896e6d0>, 9883977.134299949)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440d2fbd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440d43850>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459f8ec10>, 9883980.89052832)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440d43710>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440d80610>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458b7b070>, 9883977.85109898)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440d80b10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440e88e50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458437d90>, 9883979.4017963)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440d65d50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440e12f10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440db9c90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440ead890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440fab610>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440d11210>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458863620>, 9883976.825423451)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440fab550>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345896d0f0>, 9883976.28535946)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440d110d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458b7bd90>, 9883978.178818656)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440e13250>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458b7aba0>, 9883979.240540357)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440db9a90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458861cc0>, 9883976.488595096)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440ead590>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440dc5390>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458863d20>, 9883979.982303556)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344109ad50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440f95710>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458b78210>, 9883978.521436457)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440dc6c50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440bfded0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440c10e10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458b79710>, 9883980.049201611)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440e3ee50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440c3a190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345896f150>, 9883979.91870218)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440c3a050>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440c4cf10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440c563d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459f8f700>, 9883980.778901633)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440c55f90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440de8050>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440c80e50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345896eb30>, 9883984.555746872)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440de8650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458b7b690>, 9883980.869619155)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440c80c50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344100bd50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458745e10>, 9883980.485720528)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344100b5d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534410dacd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440cbd090>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345896ea50>, 9883980.658516267)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534410d99d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458537770>, 9883981.900097491)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440cbc390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440cd0e10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345896cfa0>, 9883979.225922544)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440bfe590>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440de1290>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440d38cd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458b7a350>, 9883982.22015246)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440c4b910>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440c42590>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345896f310>, 9883978.841942592)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440d12550>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458535780>, 9883981.442499762)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440cd0c10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458b7a5f0>, 9883978.88748422)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440d390d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458b7b540>, 9883978.963223767)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441005f50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440b6cd10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458317e70>, 9883982.44316951)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440b6cb10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440b09f50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458a70670>, 9883982.354905432)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440b082d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440d81b90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440b9cd10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458316b30>, 9883985.947138349)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440d81710>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458c9fa10>, 9883982.455210144)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440b9cb10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440bb3f10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534585349f0>, 9883982.686364535)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440bb3e10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440bd8790>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458c9fd90>, 9883985.313903917)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440bd8a90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440b11350>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458437bd0>, 9883982.63799318)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440b11050>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440b582d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440b08710>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440b1ce10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440b36710>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440cea450>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440b58290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458437620>, 9883980.995780423)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441026cd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440cde1d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458315400>, 9883981.396873988)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440b1d490>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458536f90>, 9883981.224054808)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440cdddd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458316510>, 9883981.226021668)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440ce9ad0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458314d70>, 9883981.511804763)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440b08bd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458c9f0e0>, 9883984.215372248)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440b58410>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458314de0>, 9883981.479963653)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440b58650>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440c575d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458c9da90>, 9883983.504804833)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440c4cfd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440bbfd50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458c9d320>, 9883988.659209255)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440bbdf90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440bef910>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458c9e970>, 9883985.095913742)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440bef6d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440a7c790>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440a83350>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458534c20>, 9883984.476099696)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440a7c710>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458eb3690>, 9883984.363349888)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440a80690>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440c4b710>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440aa5ed0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458eb1080>, 9883984.539573343)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440a91b50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458eb22e0>, 9883987.480530912)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440aa5cd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440ad10d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440ae6e10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440ae8710>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458eb0d70>, 9883985.237999348)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440bbd250>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440ac4790>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344090ad50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458eb1860>, 9883984.460432213)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440ac4810>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458eb2cf0>, 9883986.260516789)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440908550>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440ab4ad0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344092e0d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458314830>, 9883984.767663112)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440ab6150>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458eb2c80>, 9883985.463819953)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344092df90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440b3ae50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344094f590>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440a41c10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440a5bc10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458c9c9f0>, 9883985.9047046)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440ad0f50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458536900>, 9883984.64300757)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440ae6cd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458c9e430>, 9883994.699209241)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440b39c10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458c9dd30>, 9883987.030333249)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440b3acd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458fc3230>, 9883986.983568173)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344094fc90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458c9e740>, 9883983.455185706)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440a422d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344099ced0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458fc32a0>, 9883995.780419484)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440ac44d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534409b1f10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458eb10f0>, 9883986.471136538)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534409b2250>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440970c50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534409e4f90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458eb0980>, 9883995.007353228)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440a91010>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458fc3690>, 9883988.913247803)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534409e5490>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344098ca10>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534584347c0>, 9883986.91961487)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344098c590>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440bbd450>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458c9cfa0>, 9883987.46421478)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440bbd510>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440839c10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534590db7e0>, 9883994.873350741)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440839a10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440806690>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534590dadd0>, 9883995.393083116)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534408066d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534408508d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534590dbd20>, 9883987.611540344)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440850790>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534407fdb90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440884650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458eb1240>, 9883988.127637353)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534407fe450>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534590d81a0>, 9883995.203243444)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440884d50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534408b0c90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344098ed90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534409c2110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440a00490>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458fc0ec0>, 9883986.499120396)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534409c27d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458fc0360>, 9883988.499382768)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534408b10d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458fc1e80>, 9883994.920929374)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344098ec50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458c9f620>, 9883986.708094906)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440925150>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534409eab50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345941cc20>, 9883990.668178476)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534409eaf90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344099f950>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440761210>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345941f620>, 9883990.641913967)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440826e50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345941d400>, 9883991.960353935)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534407610d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440775910>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344079afd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458c9fbd0>, 9884002.00978315)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440775690>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534590d88a0>, 9883992.85123622)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344079ae90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534407b2290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534590c9e80>, 9883991.480436368)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534407b2910>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534407c2e10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534590c8de0>, 9883991.44605056)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534407c3190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440763350>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534408ec550>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534408ee790>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440710290>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440715090>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440748310>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534408cde10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458fc3a10>, 9883991.88524732)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534408cdc10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458fc1d30>, 9883989.538570132)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440710750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345941c980>, 9883992.354877181)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440bd0210>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345941f7e0>, 9883988.957193837)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534408ecc10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458fc15c0>, 9883989.88903417)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534408ecf10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345941fc40>, 9883990.106968043)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440edd0d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345941d390>, 9883989.965549508)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440748810>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440643ad0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534590ca2e0>, 9883993.704491034)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440643ed0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534407f1990>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534590cb310>, 9883995.176352957)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534407f1790>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440aa6ed0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440762410>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440614e90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440733f10>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534406332d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534590c8980>, 9883992.008067492)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534408ce750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534590cba10>, 9883993.000457408)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534407624d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534590caba0>, 9883992.955271065)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440615590>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534590d9d30>, 9883992.652725449)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534407e3190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534590c9a20>, 9883992.976086816)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440633110>\n",
      "Evaluating workflow:   0%|          | 1/500 [00:01<13:14,  1.59s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.3333333333333333, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   1%|          | 3/500 [00:01<04:24,  1.88it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   1%|          | 5/500 [00:02<02:48,  2.93it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   2%|▏         | 8/500 [00:02<01:52,  4.38it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   2%|▏         | 9/500 [00:03<02:24,  3.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   2%|▏         | 12/500 [00:03<01:42,  4.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.18181818181818182, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   3%|▎         | 13/500 [00:03<01:45,  4.61it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   3%|▎         | 14/500 [00:04<01:48,  4.47it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   3%|▎         | 16/500 [00:04<01:42,  4.72it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   3%|▎         | 17/500 [00:04<01:41,  4.77it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   4%|▍         | 19/500 [00:05<01:30,  5.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.3333333333333333, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.4, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   4%|▍         | 20/500 [00:05<01:20,  5.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.7272727272727273, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   4%|▍         | 21/500 [00:05<02:17,  3.49it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   5%|▍         | 23/500 [00:06<01:45,  4.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   5%|▌         | 26/500 [00:06<01:25,  5.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.75, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   6%|▌         | 29/500 [00:06<01:01,  7.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   6%|▌         | 31/500 [00:07<01:59,  3.93it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   6%|▋         | 32/500 [00:08<02:03,  3.80it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   7%|▋         | 33/500 [00:08<02:23,  3.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.4444444444444445, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   7%|▋         | 35/500 [00:09<02:26,  3.17it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.56, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   7%|▋         | 37/500 [00:09<01:50,  4.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   8%|▊         | 39/500 [00:09<01:26,  5.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   9%|▉         | 44/500 [00:10<00:42, 10.82it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.5454545454545454, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   9%|▉         | 46/500 [00:10<00:43, 10.40it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.5454545454545454, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  10%|▉         | 48/500 [00:10<00:51,  8.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  10%|█         | 52/500 [00:11<00:54,  8.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  11%|█         | 53/500 [00:11<01:44,  4.27it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.5714285714285715, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  11%|█         | 55/500 [00:12<01:26,  5.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  12%|█▏        | 58/500 [00:12<01:10,  6.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.4444444444444445, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  12%|█▏        | 62/500 [00:12<00:59,  7.37it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.33333333333333337, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  13%|█▎        | 65/500 [00:13<00:52,  8.29it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.08695652173913045, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  13%|█▎        | 66/500 [00:13<01:13,  5.93it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.30769230769230765, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  13%|█▎        | 67/500 [00:13<01:25,  5.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  14%|█▍        | 71/500 [00:14<00:58,  7.37it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  15%|█▌        | 75/500 [00:14<00:37, 11.38it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.028985507246376812, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  15%|█▌        | 77/500 [00:15<01:10,  5.97it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.8, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  16%|█▌        | 79/500 [00:15<00:59,  7.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  17%|█▋        | 83/500 [00:16<01:07,  6.17it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.6, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.3333333333333333, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.25, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.16666666666666669, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  17%|█▋        | 86/500 [00:16<00:47,  8.63it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  18%|█▊        | 88/500 [00:16<01:03,  6.48it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  18%|█▊        | 90/500 [00:17<01:25,  4.77it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  19%|█▊        | 93/500 [00:18<01:08,  5.92it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  19%|█▉        | 95/500 [00:18<01:16,  5.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  19%|█▉        | 97/500 [00:18<01:06,  6.07it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  20%|█▉        | 98/500 [00:19<01:50,  3.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.28571428571428575, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  20%|██        | 101/500 [00:20<01:36,  4.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.4, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  21%|██        | 103/500 [00:20<01:14,  5.32it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.1904761904761905, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  21%|██        | 104/500 [00:20<01:06,  5.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  22%|██▏       | 110/500 [00:20<00:35, 10.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.9090909090909091, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.16666666666666669, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  22%|██▏       | 112/500 [00:21<00:51,  7.48it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.4799999999999999, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.5454545454545454, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  23%|██▎       | 114/500 [00:21<00:53,  7.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.5, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  23%|██▎       | 115/500 [00:22<01:22,  4.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.10526315789473684, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.8571428571428571, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  24%|██▍       | 119/500 [00:22<00:55,  6.92it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.04347826086956522, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  24%|██▍       | 121/500 [00:22<01:00,  6.29it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  25%|██▍       | 123/500 [00:23<00:54,  6.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.3076923076923077, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  25%|██▌       | 125/500 [00:23<00:44,  8.42it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  25%|██▌       | 126/500 [00:23<00:45,  8.17it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  25%|██▌       | 127/500 [00:23<00:54,  6.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.7368421052631579, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  26%|██▌       | 129/500 [00:24<01:04,  5.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.2857142857142857, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  26%|██▌       | 130/500 [00:24<01:11,  5.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  26%|██▋       | 132/500 [00:24<01:14,  4.93it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  28%|██▊       | 138/500 [00:25<00:35, 10.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.2857142857142857, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.4, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  28%|██▊       | 141/500 [00:25<00:30, 11.88it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  29%|██▊       | 143/500 [00:25<00:40,  8.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  29%|██▉       | 145/500 [00:26<00:52,  6.71it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  29%|██▉       | 146/500 [00:26<00:57,  6.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  29%|██▉       | 147/500 [00:26<01:04,  5.47it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.6, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  30%|███       | 151/500 [00:27<00:44,  7.87it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.7692307692307693, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  30%|███       | 152/500 [00:27<00:57,  6.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  31%|███▏      | 157/500 [00:27<00:41,  8.27it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  32%|███▏      | 159/500 [00:28<00:56,  6.04it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.14285714285714288, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  32%|███▏      | 162/500 [00:28<00:55,  6.05it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  33%|███▎      | 163/500 [00:29<00:58,  5.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  33%|███▎      | 164/500 [00:29<01:04,  5.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  33%|███▎      | 167/500 [00:29<00:45,  7.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.5, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  34%|███▍      | 170/500 [00:30<00:47,  6.88it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.7142857142857143, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  34%|███▍      | 172/500 [00:30<00:37,  8.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  35%|███▍      | 174/500 [00:30<00:37,  8.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  35%|███▌      | 175/500 [00:30<00:45,  7.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  36%|███▌      | 178/500 [00:31<01:00,  5.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.3076923076923077, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  36%|███▌      | 179/500 [00:31<01:11,  4.47it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.09523809523809525, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  37%|███▋      | 183/500 [00:32<00:48,  6.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.3076923076923077, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  37%|███▋      | 187/500 [00:32<00:27, 11.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  38%|███▊      | 189/500 [00:32<00:34,  8.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.4, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.25, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  38%|███▊      | 192/500 [00:32<00:31,  9.92it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  39%|███▉      | 194/500 [00:33<00:44,  6.89it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.14285714285714288, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  39%|███▉      | 196/500 [00:34<00:56,  5.42it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  40%|████      | 200/500 [00:34<00:44,  6.67it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.2222222222222222, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.5, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  40%|████      | 202/500 [00:34<00:45,  6.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  41%|████      | 206/500 [00:35<00:34,  8.58it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.38095238095238093, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.7777777777777778, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  42%|████▏     | 208/500 [00:35<00:49,  5.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.19999999999999998, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.4, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  42%|████▏     | 212/500 [00:36<00:43,  6.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.4444444444444445, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  43%|████▎     | 216/500 [00:37<00:38,  7.35it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  44%|████▍     | 219/500 [00:37<00:47,  5.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.2857142857142857, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.8, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  44%|████▍     | 222/500 [00:38<00:42,  6.48it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  45%|████▍     | 224/500 [00:38<00:41,  6.72it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.6153846153846153, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  45%|████▌     | 227/500 [00:38<00:31,  8.71it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  46%|████▌     | 229/500 [00:39<00:40,  6.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.21428571428571425, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.5, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  47%|████▋     | 233/500 [00:39<00:39,  6.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  47%|████▋     | 234/500 [00:39<00:37,  7.04it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.15384615384615385, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  47%|████▋     | 237/500 [00:40<00:35,  7.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  48%|████▊     | 239/500 [00:40<00:45,  5.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  48%|████▊     | 240/500 [00:41<01:01,  4.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  48%|████▊     | 241/500 [00:41<00:59,  4.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  48%|████▊     | 242/500 [00:41<01:06,  3.88it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  49%|████▉     | 244/500 [00:42<00:55,  4.58it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  49%|████▉     | 245/500 [00:42<00:58,  4.38it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  50%|█████     | 250/500 [00:42<00:30,  8.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.3636363636363636, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.21052631578947367, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.4444444444444445, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  51%|█████     | 255/500 [00:43<00:24, 10.05it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.4, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.13333333333333336, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.022727272727272728, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  51%|█████▏    | 257/500 [00:43<00:30,  7.88it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  52%|█████▏    | 260/500 [00:44<00:47,  5.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  53%|█████▎    | 264/500 [00:44<00:29,  8.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.5454545454545454, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  53%|█████▎    | 266/500 [00:45<00:34,  6.83it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.8, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  54%|█████▍    | 270/500 [00:45<00:34,  6.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.5, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  54%|█████▍    | 272/500 [00:46<00:50,  4.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.5625, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  55%|█████▌    | 275/500 [00:47<00:38,  5.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  56%|█████▌    | 278/500 [00:47<00:35,  6.27it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  56%|█████▋    | 282/500 [00:48<00:33,  6.57it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.2857142857142857, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  57%|█████▋    | 283/500 [00:48<00:47,  4.61it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.2857142857142857, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  57%|█████▋    | 284/500 [00:48<00:46,  4.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  57%|█████▋    | 286/500 [00:49<00:49,  4.35it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  58%|█████▊    | 288/500 [00:49<00:39,  5.40it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.23529411764705882, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.625, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  58%|█████▊    | 292/500 [00:49<00:25,  8.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.5714285714285715, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  59%|█████▉    | 294/500 [00:50<00:33,  6.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.8, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.3076923076923077, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  59%|█████▉    | 296/500 [00:50<00:36,  5.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.4, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.8571428571428571, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  60%|█████▉    | 299/500 [00:50<00:26,  7.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  60%|██████    | 301/500 [00:51<00:29,  6.78it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  61%|██████    | 304/500 [00:51<00:27,  7.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  61%|██████    | 305/500 [00:52<00:31,  6.17it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.5, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.4444444444444445, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  61%|██████▏   | 307/500 [00:52<00:45,  4.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  62%|██████▏   | 312/500 [00:53<00:25,  7.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.47619047619047616, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  63%|██████▎   | 314/500 [00:53<00:25,  7.16it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.6666666666666665, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  64%|██████▎   | 318/500 [00:53<00:20,  8.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.8, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  64%|██████▍   | 322/500 [00:54<00:22,  8.06it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.8, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.5714285714285715, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  65%|██████▌   | 325/500 [00:54<00:25,  6.83it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  66%|██████▌   | 329/500 [00:55<00:26,  6.55it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.5714285714285715, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.8, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  67%|██████▋   | 335/500 [00:56<00:17,  9.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.3157894736842105, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.25, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.5454545454545454, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  67%|██████▋   | 337/500 [00:56<00:17,  9.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  68%|██████▊   | 340/500 [00:56<00:21,  7.60it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  68%|██████▊   | 342/500 [00:57<00:17,  9.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.8, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  69%|██████▉   | 344/500 [00:57<00:28,  5.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.4, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  69%|██████▉   | 347/500 [00:58<00:25,  5.92it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  70%|██████▉   | 348/500 [00:58<00:25,  5.92it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.8, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  71%|███████   | 353/500 [00:58<00:16,  9.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.33333333333333337, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  71%|███████   | 355/500 [00:58<00:14,  9.92it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.3333333333333333, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  71%|███████▏  | 357/500 [00:59<00:12, 11.32it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.5, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  72%|███████▏  | 359/500 [00:59<00:15,  9.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  72%|███████▏  | 361/500 [00:59<00:16,  8.61it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.8, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.045454545454545456, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  73%|███████▎  | 363/500 [00:59<00:15,  8.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.6363636363636364, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  73%|███████▎  | 365/500 [01:00<00:20,  6.58it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.5714285714285715, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  73%|███████▎  | 367/500 [01:01<00:28,  4.66it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  74%|███████▎  | 368/500 [01:01<00:26,  4.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  74%|███████▍  | 369/500 [01:01<00:26,  4.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.8, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  75%|███████▍  | 373/500 [01:01<00:15,  8.07it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.75, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  75%|███████▌  | 376/500 [01:02<00:15,  7.97it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  75%|███████▌  | 377/500 [01:02<00:17,  7.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.05714285714285715, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  76%|███████▌  | 379/500 [01:02<00:22,  5.40it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.26666666666666666, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  76%|███████▌  | 380/500 [01:03<00:25,  4.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  77%|███████▋  | 383/500 [01:03<00:18,  6.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.631578947368421, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  78%|███████▊  | 388/500 [01:04<00:20,  5.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.5, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  79%|███████▊  | 393/500 [01:04<00:14,  7.17it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  79%|███████▉  | 395/500 [01:05<00:13,  7.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  80%|███████▉  | 398/500 [01:05<00:13,  7.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  80%|████████  | 400/500 [01:05<00:13,  7.38it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.5714285714285715, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.5, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  80%|████████  | 402/500 [01:06<00:11,  8.71it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  81%|████████  | 403/500 [01:06<00:12,  7.74it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  81%|████████  | 405/500 [01:06<00:15,  6.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.18181818181818182, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  81%|████████▏ | 407/500 [01:06<00:13,  7.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  82%|████████▏ | 410/500 [01:07<00:12,  7.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.3333333333333333, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  82%|████████▏ | 412/500 [01:07<00:10,  8.74it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.33333333333333337, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.3, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  83%|████████▎ | 415/500 [01:07<00:11,  7.61it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.8, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.8, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.5, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  83%|████████▎ | 417/500 [01:08<00:15,  5.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  84%|████████▍ | 419/500 [01:09<00:17,  4.60it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  84%|████████▍ | 422/500 [01:09<00:10,  7.58it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  85%|████████▍ | 424/500 [01:09<00:09,  7.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.4, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  85%|████████▌ | 426/500 [01:10<00:13,  5.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  86%|████████▌ | 430/500 [01:10<00:08,  8.17it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.5, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.8571428571428571, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.2857142857142857, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  86%|████████▋ | 432/500 [01:10<00:10,  6.42it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.5714285714285715, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  87%|████████▋ | 433/500 [01:10<00:11,  5.92it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  87%|████████▋ | 436/500 [01:11<00:09,  6.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  87%|████████▋ | 437/500 [01:11<00:10,  6.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  88%|████████▊ | 439/500 [01:12<00:12,  5.04it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.0909090909090909, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  89%|████████▊ | 443/500 [01:12<00:07,  7.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.8235294117647058, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.5454545454545454, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  89%|████████▉ | 445/500 [01:12<00:05,  9.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  89%|████████▉ | 447/500 [01:13<00:10,  4.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.4444444444444445, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  90%|████████▉ | 448/500 [01:13<00:09,  5.35it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.4444444444444445, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  90%|█████████ | 450/500 [01:14<00:10,  4.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.2857142857142857, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  90%|█████████ | 451/500 [01:14<00:09,  5.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.6153846153846153, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  91%|█████████ | 454/500 [01:14<00:06,  6.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  91%|█████████ | 455/500 [01:14<00:07,  5.83it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  91%|█████████▏| 457/500 [01:15<00:08,  5.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.5, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  92%|█████████▏| 458/500 [01:15<00:08,  5.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  92%|█████████▏| 459/500 [01:15<00:09,  4.35it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.4444444444444445, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  92%|█████████▏| 461/500 [01:16<00:09,  4.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  93%|█████████▎| 466/500 [01:16<00:04,  7.77it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  94%|█████████▍| 469/500 [01:16<00:03,  8.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.47619047619047616, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  94%|█████████▍| 471/500 [01:17<00:04,  6.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.5263157894736842, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  95%|█████████▍| 473/500 [01:17<00:03,  6.89it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  95%|█████████▌| 476/500 [01:17<00:02,  8.63it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  96%|█████████▌| 478/500 [01:18<00:03,  6.27it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.4, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.33333333333333337, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  96%|█████████▌| 480/500 [01:18<00:03,  5.93it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.8, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.2857142857142857, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  97%|█████████▋| 485/500 [01:19<00:01,  7.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.5, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.5714285714285715, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  97%|█████████▋| 487/500 [01:19<00:01,  9.36it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.8, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  98%|█████████▊| 490/500 [01:20<00:02,  4.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.33333333333333337, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  99%|█████████▊| 493/500 [01:20<00:00,  7.40it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.375, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  99%|█████████▉| 495/500 [01:21<00:00,  5.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.33333333333333337, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.10526315789473684, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow: 100%|█████████▉| 498/500 [01:23<00:00,  2.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.05, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow: 100%|██████████| 500/500 [01:30<00:00,  5.52it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   1%|          | 3/500 [00:01<04:20,  1.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.3333333333333333, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   2%|▏         | 8/500 [00:02<01:27,  5.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.18181818181818182, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   3%|▎         | 13/500 [00:02<01:08,  7.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.75, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   3%|▎         | 15/500 [00:03<01:24,  5.71it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   3%|▎         | 17/500 [00:03<01:22,  5.87it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   4%|▍         | 19/500 [00:03<01:07,  7.17it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   4%|▍         | 20/500 [00:04<01:22,  5.82it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   5%|▍         | 23/500 [00:04<01:02,  7.58it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.4, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.7272727272727273, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   5%|▌         | 25/500 [00:04<01:03,  7.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   5%|▌         | 27/500 [00:05<01:12,  6.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   6%|▌         | 28/500 [00:05<01:25,  5.49it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   6%|▌         | 30/500 [00:05<01:17,  6.09it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   7%|▋         | 33/500 [00:06<01:25,  5.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.3333333333333333, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   7%|▋         | 36/500 [00:06<01:15,  6.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.8571428571428571, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   8%|▊         | 38/500 [00:07<01:07,  6.83it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   8%|▊         | 39/500 [00:07<01:08,  6.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   8%|▊         | 40/500 [00:07<01:39,  4.63it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   9%|▉         | 46/500 [00:08<01:07,  6.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  10%|▉         | 48/500 [00:08<01:05,  6.89it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.5714285714285715, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  10%|▉         | 49/500 [00:09<01:18,  5.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  10%|█         | 50/500 [00:09<01:34,  4.74it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.4444444444444445, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.43750000000000006, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  11%|█         | 55/500 [00:10<01:08,  6.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metricsmetrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      " {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.3076923076923077, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  12%|█▏        | 58/500 [00:10<01:14,  5.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  12%|█▏        | 59/500 [00:10<01:10,  6.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  12%|█▏        | 60/500 [00:11<01:19,  5.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  13%|█▎        | 63/500 [00:11<01:19,  5.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.08695652173913045, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  13%|█▎        | 66/500 [00:12<01:29,  4.86it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  14%|█▍        | 71/500 [00:12<01:04,  6.66it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  15%|█▍        | 74/500 [00:13<01:06,  6.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  15%|█▌        | 75/500 [00:13<01:10,  6.06it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.8, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  15%|█▌        | 76/500 [00:13<01:15,  5.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.33333333333333337, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  16%|█▌        | 79/500 [00:14<01:30,  4.63it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.3333333333333333, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  16%|█▌        | 81/500 [00:14<01:06,  6.27it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.33333333333333337, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  17%|█▋        | 83/500 [00:15<01:00,  6.88it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  17%|█▋        | 86/500 [00:15<01:24,  4.93it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.6, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  17%|█▋        | 87/500 [00:16<01:24,  4.89it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.10526315789473684, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  18%|█▊        | 92/500 [00:16<00:58,  6.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  19%|█▉        | 94/500 [00:16<00:47,  8.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  19%|█▉        | 96/500 [00:17<00:59,  6.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.3157894736842105, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  20%|█▉        | 98/500 [00:17<01:08,  5.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  20%|██        | 100/500 [00:18<01:15,  5.27it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.4, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.1904761904761905, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  21%|██        | 103/500 [00:18<00:44,  8.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  21%|██        | 105/500 [00:19<01:40,  3.92it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.9090909090909091, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  22%|██▏       | 109/500 [00:19<01:12,  5.42it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.27272727272727276, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  22%|██▏       | 110/500 [00:20<01:12,  5.35it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  23%|██▎       | 113/500 [00:20<01:00,  6.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.4799999999999999, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.5, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534406b9c10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534590d9be0>, 9883996.300823541)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534406b9e90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440558a50>\n",
      "Evaluating workflow:  23%|██▎       | 114/500 [00:21<01:32,  4.19it/s]Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440574cd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458ff2900>, 9883996.029327312)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440575390>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458ff3c40>, 9883995.732334644)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440559110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440427d90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458201940>, 9883998.162508752)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440479590>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440452750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534582038c0>, 9883998.188517235)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440453ad0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440638750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534404c9e10>, 9883999.543715104)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344031be90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534404bf7d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534404ca890>, 9884000.145295188)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440354950>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344048bad0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534404af250>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458203c40>, 9883998.32624106)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534404af110>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534582011d0>, 9883998.325056704)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344048b990>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534406d6ed0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458eb0bb0>, 9884001.375082497)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344039b290>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440894a50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534403c0fa0>, 9884003.42503614)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440529c10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534402bd9d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534402a4de0>, 9884003.001349645)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534402bd8d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440273a10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534404cbbd0>, 9884006.552377287)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440272f50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534403dcb10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440208590>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534403a5110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534402288d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534403c1710>, 9884001.042343099)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534401fa610>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534403c0de0>, 9884000.85207798)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534403dec90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534403c1e80>, 9884001.254591027)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440351350>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534403c18d0>, 9884000.61686147)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534403a5250>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440232710>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534402a64a0>, 9884004.354836958)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440123f50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534400fbcd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534402a7770>, 9884006.163621703)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440161c10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534401a01d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534403c2c10>, 9884004.895099428)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534401a0290>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534405bd7d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344000c7d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344001b710>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534402a5550>, 9884003.98406914)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534405bdb90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440182090>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534591f0050>, 9884004.120627703)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344000cf10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534401922e0>, 9884004.069601385)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440019610>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534403c20b0>, 9884004.362319876)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440182b10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534401348d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153440191a90>, 9884005.459517198)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534405bd410>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534400672d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153440088fa0>, 9884007.453446884)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440067890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534400d9ed0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534400898d0>, 9884007.146252455)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534400da150>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534400d87d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15344008b000>, 9884007.601036064)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440045e90>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344004de10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343ff638d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534400aa010>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534402a7c40>, 9884007.217737235)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534400bd910>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15344008bc40>, 9884006.763661793)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343ff63790>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153440088590>, 9884006.520387568)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534400aa510>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343fe22710>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534404cb9a0>, 9884009.927198783)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343fee6f90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343ffec5d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534403c2270>, 9884008.588139564)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343ffec3d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343fc2b410>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343fd21b90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343fd0f6d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343ff3ec50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343ff7f2a0>, 9884010.734775065)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343fd0d350>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343ff7fd90>, 9884011.356122443)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343fd91c10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343fc58910>, 9884014.281071052)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343fc2bfd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343fe660b0>, 9884011.211206544)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343fd21d50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343fb60390>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343ff7c600>, 9884014.979884332)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343fc74150>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343fcca410>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343fc5bbd0>, 9884015.435474308)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343fcc90d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343fc5f450>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343fb5de10>, 9884015.704809852)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343fc5f150>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343fc9f5d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343fcc11d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343fc594e0>, 9884013.538706308)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343fc9f450>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343fc59fd0>, 9884013.603914052)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343fe81e90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344013e550>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343fb5f070>, 9884016.428139515)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343fa07d50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343fa76d50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343fa3a820>, 9884020.078333825)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343fcc3190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343fa8fad0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343fa3b700>, 9884018.26762598)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343ffd0ad0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343fa62dd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343fc9d190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345941f230>, 9884016.1116952)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343fa637d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343fb5f230>, 9884016.141780844)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343fc9d850>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343fbd2e10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343f932660>, 9884019.468257513)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343fbd3010>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f800bd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343f82cc20>, 9884020.17605177)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f958890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343fb97e90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f9a07d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f9b6d90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343fa3b3f0>, 9884017.836785672)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f923290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343f932040>, 9884018.55621153)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f9a05d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343f932820>, 9884019.135478688)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f9b74d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f70fd10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343f931550>, 9884025.837690316)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f8141d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f73a850>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343f719010>, 9884022.946244936)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f73ab90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f7b8c90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343f718a60>, 9884022.105285753)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f7b8bd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f8c1790>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f848f90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f738690>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f75a8d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f77ded0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343f82e9e0>, 9884021.965944314)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f8c1590>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343f930c20>, 9884021.059585562)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343fd62ad0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343fd6a6d0>, 9884021.814388756)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343fcf15d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343f7192b0>, 9884022.125406032)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f75a510>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343f719b00>, 9884021.76962143)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f77e610>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f8ed250>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f823f90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f639a10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f8756d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343fd30f90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343ff61450>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534404ce8d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534405be9d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440839910>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440c61250>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440b95050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459d77930>, 9884039.971498368)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f8ec1d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459d74a60>, 9884044.918614788)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f6136d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345bae3d20>, 9884041.90806224)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f638a50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345bae25f0>, 9884040.537023196)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f876910>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345962fbd0>, 9884042.538213598)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343fd31690>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459f8fd20>, 9884043.513493214)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344002f750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459a39320>, 9884048.605416942)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534404ce810>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a19aeb0>, 9884039.894742021)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534405bd1d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459a3af90>, 9884041.55649458)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344083ae90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440c73bd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440c403d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440bf8710>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440f64590>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345924e490>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345901e8d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458333790>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345942c1d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345a722d90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a3e33f0>, 9884048.21957688)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153443ff8450>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a3e19b0>, 9884040.214874027)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153443f1c590>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a4dce50>, 9884040.22712568)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345bb1ecd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345993cec0>, 9884048.277192432)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440c7f8d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534423c75b0>, 9884040.893500226)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440c416d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a3b6190>, 9884048.292802168)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440bfb0d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15344208f7e0>, 9884040.925031794)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440f647d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1533fadede80>, 9884041.046226555)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345904acd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a3e3ee0>, 9884048.198261917)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153459016110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344067e1d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440699d90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344069bd10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534406b9ed0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534406d5150>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534406f1890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344061e510>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440512d90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344051add0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440528fd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344054c090>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440593790>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534405c6f90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344084b890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534405e0910>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440538e10>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440406e50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344041a7d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440846110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440441e90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440464450>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534405bc2d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440473350>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440318b90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440352890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440368790>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534403bc310>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534402fc650>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440310a90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534403471d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440443250>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440309d10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440723310>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440259750>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344038a390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344028b1d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440352a10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440208bd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440114dd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440117790>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344013e110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440186090>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344027c890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534401ddad0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440208f10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344005cf90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534401a3c90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534400a8e50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343ff01790>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344008c350>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343ff21390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343ff2c0d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343ff50890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343ff6dbd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343ff9e710>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343ff93750>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343ff6cb10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343fe03d90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343fe146d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343fe30ad0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343ff2a910>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343fe474d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343fe78450>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534400a9ed0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343fe90c50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343fea4650>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343fea6610>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343fecc650>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343ffeedd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343fd05250>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343fd2d710>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343fd63490>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343fd84ad0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343fd8be50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343fdc9e50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343fdea2d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343fdebf50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343fc1e8d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343fc27650>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343fc31350>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343fc52990>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343fc5e450>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343fc9c490>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343fcc2e10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343fb27110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343fb74510>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343fb8e990>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343fbac210>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343fbc06d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343fce9710>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343fb0ead0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343fb52590>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343fbe5010>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343fa20950>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343fa34990>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343fa6d510>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343fbf15d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343fa9b910>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343fab76d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343fad8910>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343fbc0050>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f9218d0>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f938b50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f9621d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f98c1d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f9e2e90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343fc279d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f8370d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f866110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f875e90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f970790>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f823510>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f877d10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f8865d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f8d9250>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f7297d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f7aa510>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f7c9350>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f6fe090>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f7a8b90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459a3ba80>, 9884041.97674602)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440c62990>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459a39e80>, 9884048.397893475)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440ae37d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534591f3460>, 9883994.035598569)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344067e950>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458437f50>, 9883994.138980577)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440699c50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534590c9710>, 9883995.224813068)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344069b8d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534591f0ec0>, 9883993.98077066)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534406b9cd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534591f34d0>, 9883997.071694903)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440615b50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534590cb0e0>, 9883995.349097366)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534406f1750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534591f1cc0>, 9883997.17207062)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344061d490>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534591f00c0>, 9883995.38856411)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440513450>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534591f2900>, 9883995.7000849)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534406da910>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458eb0130>, 9883995.529092057)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440528e90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458ff00c0>, 9883997.450191695)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344054ead0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458ff3ee0>, 9883997.50662504)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440593650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458ff2ac0>, 9883997.462308384)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534405c7650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534591f0750>, 9883998.194368452)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534405c7c10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534590da890>, 9883997.989328839)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534405e0510>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458ff2350>, 9883997.35525227)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534405393d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458ff0ec0>, 9883997.272653207)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534405b7450>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534590c9940>, 9884013.756523268)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344041a690>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458203e00>, 9883997.696998049)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440838910>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534582023c0>, 9883997.59757008)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440441c90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534591f2ba0>, 9883997.267345864)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344045bb10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458316a50>, 9883998.810795171)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534406aa750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458534590>, 9884004.18983124)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534404739d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534404c9c50>, 9883999.47769104)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534405591d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534404ca6d0>, 9884000.555283088)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440352750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534404caeb0>, 9884000.163911588)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440368590>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534403c03d0>, 9884000.352786068)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534403bc810>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534582029e0>, 9883999.758496013)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440308910>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458200fa0>, 9884000.237067616)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534405d6e50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534590cbd90>, 9883999.81253583)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440347090>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534591f0fa0>, 9884001.55411604)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440442d90>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534582018d0>, 9884000.656561064)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344030a5d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458ff3150>, 9884002.158481924)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534403476d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534403c2d60>, 9884002.191719407)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440259e10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534403c3a10>, 9884002.294416623)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534405bc4d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534403c3c40>, 9884002.331568968)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344028aed0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534404cb850>, 9884002.380318392)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440352610>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534404c9470>, 9884003.588450072)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440208a90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534402a6040>, 9884003.955787633)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534405bd3d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534402a6270>, 9884006.345637793)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534401173d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534402a6d60>, 9884004.620619062)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344013dd50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534404c8910>, 9884004.749479735)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440185e90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153440190d00>, 9884004.382439)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344027c390>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534404c8ad0>, 9884005.017703066)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534401de290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153440190980>, 9884005.415696628)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440209450>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534402a5d30>, 9884006.762425948)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440345550>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153440191780>, 9884004.533896867)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440399cd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534403c0210>, 9884007.522507124)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534400ab290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15344008a3c0>, 9884008.286357664)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343ff01250>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153440192eb0>, 9884007.423645915)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344008c550>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15344008a6d0>, 9884007.921944609)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343ff21110>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458c9d1d0>, 9884007.247976406)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343ff2c790>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15344008b690>, 9884006.756239863)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343ff50690>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534402a6ba0>, 9884008.734977685)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343ff6da10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343ff7cc20>, 9884007.5698236)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343ff9e5d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343ff7c9f0>, 9884008.322519172)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343ff93e50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153440193e00>, 9884010.277869452)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343ff6c890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343ff7d240>, 9884009.130027993)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343fe03c50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343ff7e970>, 9884012.66173602)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534400b8050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343ff7f4d0>, 9884009.826566743)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343fe31210>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343ff7f700>, 9884009.383491628)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343ffb3b90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343ff7f8c0>, 9884009.17975742)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344008fc90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343fe64590>, 9884010.727468101)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343fe78950>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343fe649f0>, 9884010.171123082)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343fe6c510>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153440088280>, 9884008.992342912)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343fe90a50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153440088f30>, 9884009.702918865)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343fea4450>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343fe656a0>, 9884010.603200693)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343fea6390>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343fe65c50>, 9884009.841412276)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343fecc450>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15344008af90>, 9884012.261192894)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343ffeee10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343fe66a50>, 9884011.945751874)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440267990>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343fe67310>, 9884010.321882002)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343fd2de50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343fd683d0>, 9884012.7052579)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343fd63350>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534401933f0>, 9884012.629867584)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343fd84f90>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343fd69390>, 9884012.72778718)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343fd8bd50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343fd69ef0>, 9884011.2645874)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343fdca510>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343fd6aac0>, 9884017.370919943)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343fdea850>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343fd6af20>, 9884012.581983617)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343fdb3190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343fd6b0e0>, 9884012.207235996)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343fc1ee50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343fd6b380>, 9884015.122322405)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343fe47c90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343fd6a270>, 9884012.653611472)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343fc31050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343fc581a0>, 9884012.707520736)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343fc525d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343fc584b0>, 9884012.860243171)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534400f9d90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343fc59320>, 9884013.537564682)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343fc8b190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343fc58c20>, 9884015.58238356)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343fcc3050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343fd6bee0>, 9884014.978464443)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343ff3e190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343fb5cd70>, 9884014.893663596)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343fb74c50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343fb5d240>, 9884018.401409868)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343fb8ee50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343fb5dc50>, 9884015.056488557)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343fbac750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343fb5e040>, 9884016.089815296)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343fc28710>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343fc5a6d0>, 9884014.138543798)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440149f90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343ff7fa80>, 9884014.776351554)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343fb0e910>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153440088d70>, 9884014.789835384)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343fb52450>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343fb5e740>, 9884020.132526696)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343fbe54d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343fb5f8c0>, 9884016.563538648)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343fa20790>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343fb5fe70>, 9884016.553870566)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343fa35050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343fa38c90>, 9884016.798382519)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343fb7db50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343fa39b00>, 9884017.22373718)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343fe30ed0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343fa39d30>, 9884017.483397428)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343fa9b7d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343fa3a3c0>, 9884017.570038984)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343fab7590>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343fa3a9e0>, 9884017.40613044)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343fc47cd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343fa3b460>, 9884019.746420177)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343fbc05d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343fa3bbd0>, 9884017.455562256)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f9216d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343fb5f5b0>, 9884018.911911812)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f938950>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343fa3acf0>, 9884018.955122646)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f962550>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343f931a20>, 9884019.383692307)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f98c890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343f933150>, 9884019.76369467)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343fdbc690>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343fa3a5f0>, 9884019.36211206)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343fc27e10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343fa3b9a0>, 9884020.126856862)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f836c50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343f82ce50>, 9884021.817483006)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f859c10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343f82d400>, 9884022.319626924)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343fc5e4d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343f931780>, 9884019.344796851)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f970710>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343fa3a660>, 9884019.448413841)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f823c10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343ff7c830>, 9884020.984928576)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f8778d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343f930520>, 9884024.302122623)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f93ed10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343f82db00>, 9884022.227005811)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f8d9110>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343f718590>, 9884022.413057905)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f729f10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343f71a7b0>, 9884022.68458851)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f822590>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343fb5e7b0>, 9884024.250939928)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f7c8f10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343f7198d0>, 9884022.281324957)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f6fe190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343f71a580>, 9884037.400303248)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f7a9350>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343fdd97d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343fe31610>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458b78ad0>, 9884043.632934684)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343fc30350>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458b7a0b0>, 9884044.203337416)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343fe31cd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343ffff150>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344040b550>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534588639a0>, 9884042.822039517)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344040bd10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343fea54d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534401b8250>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458863150>, 9884042.96020394)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534406e6f50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458b79550>, 9884043.163899468)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534401b8350>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f7e0e10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458b782f0>, 9884043.385586835)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f739c10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344025b910>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344021db90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458862970>, 9884044.517341357)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440065c50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458b7a040>, 9884042.054833028)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344025a590>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458860210>, 9884042.292845882)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440399090>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534405c6990>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534407335d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458747d90>, 9884045.033074358)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534405c5990>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440884650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458745b00>, 9884047.356419057)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440887dd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440761a10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534407a35d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458747a80>, 9884046.831597025)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440761590>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458747d20>, 9884046.117558748)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534407a3fd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440884850>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534402a5f60>, 9884063.559294233)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534408868d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f823e50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440a82310>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345863e120>, 9884045.987705292)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f822510>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345863fc40>, 9884050.36860101)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440a81c50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344081a110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534408fbf90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345863e580>, 9884047.84406601)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534409449d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534404c8520>, 9884048.13471138)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534408f8ad0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343fbd28d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343fd52510>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440478a50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534408cd750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458744f30>, 9884048.022252906)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440643610>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534407480d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458746a50>, 9884044.989779511)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344074aed0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345863d710>, 9884048.52909774)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344094c3d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534407f1a90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458744bb0>, 9884044.492964853)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343fd53610>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534587458d0>, 9884044.225062508)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344047b650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458747540>, 9884044.755764384)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344047b710>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458745f60>, 9884045.155814208)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534407f3510>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440ae01d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345863e820>, 9884048.984389788)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440ae0310>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440a41c90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459d75550>, 9884049.250267887)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440a43190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534408d9390>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459d76ba0>, 9884048.907442162)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344099ee50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440bad610>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459d775b0>, 9884050.130026758)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440884050>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440938390>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459d742f0>, 9884049.4091924)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344093b550>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440b71e10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440b5bc50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459d741a0>, 9884051.203924263)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440b73190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345bae3ee0>, 9884050.79122804)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440ae9ed0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440839410>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440a16bd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345863ff50>, 9884050.63901169)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440838a10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459d75ef0>, 9884050.774297856)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440a17550>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440bb3190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345863e2e0>, 9884053.726756608)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440bb2dd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440cccc50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440bbe6d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534408cd410>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440ae4f50>\n",
      "Evaluating workflow:  24%|██▍       | 120/500 [00:21<00:37, 10.09it/s]Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534408387d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440a7e250>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440ac7c90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345863d4e0>, 9884048.765129868)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f6bc4d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459d749f0>, 9884048.968911042)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440ac6e90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345bae0600>, 9884051.461225696)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440cccb90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459a3a580>, 9884052.815636609)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440b7da90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345863cc90>, 9884046.405262213)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440826450>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345863fcb0>, 9884047.53863666)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440ae5850>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458b78980>, 9884047.645692697)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440a7fa90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440600d50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459a387c0>, 9884050.179035628)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440850a50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440dbfe90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459a3a6d0>, 9884052.549407093)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440dbe050>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440dbb590>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534587472a0>, 9884050.62426473)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440dbb650>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440eca650>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440ed7750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459a3aa50>, 9884052.186217448)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440ec9710>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459a38ec0>, 9884051.894039787)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440ed7410>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440c990d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440e32790>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459301e10>, 9884052.530313272)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344000d0d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459d758d0>, 9884051.379174894)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440e33cd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344081b390>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459b580c0>, 9884052.755410368)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344081ad10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440da12d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345bae3b60>, 9884056.240540847)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440a03f90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440c9ba90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459b59da0>, 9884052.555953125)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440c29a90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440fc8450>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459b5b1c0>, 9884052.694783213)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440fc85d0>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440bf9bd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440d00850>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440c48310>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440c7ddd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440c838d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459a38910>, 9884050.798434228)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440643e90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459a3af20>, 9884051.010172255)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440c7cad0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534580f3e00>, 9884054.611521434)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440bf9b10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534580f1080>, 9884052.684692776)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440d00610>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459a3ac10>, 9884051.539837647)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440c48c90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440d8d650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534580f22e0>, 9884060.47811204)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440d8fdd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440b29750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459a3a2e0>, 9884054.500332305)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440b2b9d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441024250>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459d75630>, 9884056.344181076)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534593c0610>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440dfa3d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345ad5d4e0>, 9884053.736368384)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440dfaf50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534410698d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345ad5d470>, 9884054.48206306)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441068050>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440f8c4d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345ad5d8d0>, 9884055.233931623)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440f8edd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440f729d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440f94c10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440f05b50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345ad5d630>, 9884054.712644158)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440f948d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345ad5f620>, 9884056.929698467)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440f06950>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440fe8190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153459519c50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1533fae18f30>, 9884058.6741725)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153459519890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441064110>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1533fae19400>, 9884054.669459945)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440f2ba90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343fd7d850>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441036190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440bfce90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440d43950>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440e0a090>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345ad5e270>, 9884054.591859214)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440bed350>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458745860>, 9884053.487251772)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440bfd750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459a3b150>, 9884055.011051333)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440beee50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345ad5d240>, 9884053.602749849)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440e09c50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1533fae1a890>, 9884054.483840838)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441000250>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1533fae1ad60>, 9884055.944333749)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441037590>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534580f1710>, 9884052.660416933)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440d41210>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344104db50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1533fae19b70>, 9884057.692304848)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344104d6d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440ec2490>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345912cfd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1533fae1aac0>, 9884060.464794053)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534409b0950>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1533fae18590>, 9884056.807688735)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345912c450>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344109e750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15346030af90>, 9884058.565032247)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344109e3d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345919d790>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458e25290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534580f1b70>, 9884057.758096581)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441081090>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459a3ae40>, 9884060.759822747)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458e27790>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458efd250>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534592fb110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458e8b690>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153443f18b40>, 9884056.297834411)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440fb2990>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15346030b7e0>, 9884058.136927208)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458e88d90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458f9d3d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153443f186e0>, 9884058.719084635)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458f9da10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458fda310>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440b28d10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15346030aeb0>, 9884057.04984336)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f6c7850>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344118f110>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459d75b00>, 9884058.971396772)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534591bcad0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534410049d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345aa122e0>, 9884056.644756887)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458efe8d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441a477e0>, 9884058.433813548)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534597a9c10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153443f18210>, 9884059.643133046)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344118f490>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1533fae19ef0>, 9884055.684915096)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441005410>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534411caf10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441a46d60>, 9884059.585167006)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534411c9590>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534411a2e10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441a47380>, 9884058.943359653)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534411a1310>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458ed1990>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458b78bb0>, 9884058.947906006)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458ed1e90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458b92e10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458adc9d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458eb37e0>, 9884060.560180482)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458add9d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534588d8390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458bab110>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345ad5da20>, 9884061.959282177)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458bab390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458a9c850>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458951650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458eb3af0>, 9884062.312409094)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458952f10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345885ba90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534588383d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458efc910>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153459080bd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345815fc90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458cccc90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458eb3d90>, 9884059.888152324)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458bc4210>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345ad5dda0>, 9884061.1801799)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534588d8490>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441a458d0>, 9884059.102900555)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534595c2010>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345ad5ca60>, 9884061.371918876)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458a9dd90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441a446e0>, 9884058.921277076)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153459083b90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458eb3700>, 9884060.500735251)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458b0b990>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458eb1080>, 9884061.147463009)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458838950>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15346030a430>, 9884058.332376117)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458efc9d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441a47cb0>, 9884059.354007248)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458cccbd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534591703d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a2b1a20>, 9884063.367817033)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153459170210>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534591b0fd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458360b50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441a475b0>, 9884063.348855382)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153459293a10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441a46ba0>, 9884066.061389674)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458360f90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534588b3d10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345863def0>, 9884063.330526948)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534588b07d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458a6f310>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a2b0c20>, 9884061.408022048)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458a6d010>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458a4c890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a2b0de0>, 9884061.506748293)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458616c10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458951b50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534581b7a50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441a44590>, 9884063.055263199)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534589b7810>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1533fae1b380>, 9884062.576770524)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534581b4190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534410ba110>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a2b00c0>, 9884063.499683265)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458e92950>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534584c5b10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534587ebc50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a3e0f30>, 9884063.462983817)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458221510>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a3e0bb0>, 9884063.643182367)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345888d150>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458da0d50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458da3210>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534595880d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534588ef1d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a2b1a90>, 9884061.644987324)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458a44310>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458eb07c0>, 9884062.93866208)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458da1cd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153443f19a20>, 9884060.961553484)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458fa4f10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a2b2190>, 9884061.879322512)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534588ef150>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153459e41890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a2b1240>, 9884065.422354417)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345f59b910>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153459aca3d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a3e2270>, 9884064.072236732)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153459435ed0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153443fc9550>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458450150>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a3e3230>, 9884063.923759984)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441780e50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a3e2c10>, 9884064.528589794)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458450990>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1533fad46fd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a3e1320>, 9884064.045822743)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1533fad45fd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345d49bc50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441c5fe70>, 9884065.271674763)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345d498490>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458331b50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441c5d940>, 9884064.756271992)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458330150>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153459f63fd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441c5fa80>, 9884065.2778588)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153459cc9510>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153459d9be10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458d22250>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153443fe6e50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a2b3540>, 9884069.73691758)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534583ae750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441c5db70>, 9884065.25297356)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534585b3190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345a600750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a2b0c90>, 9884065.821402865)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345a3e5850>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458128610>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153459634950>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153443f8c790>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345838dcd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1533fae18c20>, 9884065.0310743)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153459d9ae90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441c5d160>, 9884065.7959922)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153443fc8fd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441c5cc20>, 9884066.720318291)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153459637290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458eb0ad0>, 9884063.097999904)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153443f8d650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a3e2580>, 9884063.42134424)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345838ee50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153443f8e350>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441c5ea50>, 9884068.268898856)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534598f44d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345960b310>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a2b2cf0>, 9884066.809721164)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153459609bd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345d327e90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534591f27b0>, 9884066.55383153)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345d3263d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345a6efa90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458ff3230>, 9884067.215575833)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345a6ee610>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153459683f10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a5f2660>, 9884067.776710376)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345a40e010>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153459d986d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153443fcba90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a3e37e0>, 9884067.42878011)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153459d9ad90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a5f2f20>, 9884066.801023645)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1533fadbc4d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345d56de50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a2b3d90>, 9884068.98861554)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345d70f690>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345f65ff50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441c5ed60>, 9884067.691721275)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345f65cc10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345a60ecd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345d4d7e50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a5f3d90>, 9884067.975239124)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458ec2c90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153443e0d5d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441c5e430>, 9884068.437963834)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345ad7d490>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345d6f7d10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345a68b7d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458ea1e50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345a852d10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a5f1010>, 9884068.738098308)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345d4d6750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a5f2f90>, 9884065.816976996)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345a8524d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345941e3c0>, 9884068.585100943)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345d6f7ed0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458eb1e80>, 9884066.188614141)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153459677550>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441a463c0>, 9884065.752446298)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458ea0890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345d534350>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345941d6a0>, 9884067.783382308)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534411c8dd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345a810a90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345941c2f0>, 9884070.616848009)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345a066090>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345ad27950>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345941fa80>, 9884068.397087775)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345a600790>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458e4ea50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1533fade6910>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441c5c3d0>, 9884069.69571876)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458e4c210>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534591f0910>, 9884070.274614217)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1533fade4910>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345f66c590>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153443113050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345941ca60>, 9884071.113540852)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153443110ed0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345a670910>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534590d9630>, 9884070.605437735)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345a6738d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442f1ca90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534590db4d0>, 9884071.085922591)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534584aca90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345bcac390>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534590d9e10>, 9884073.577165226)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345bcaf950>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345bbc9790>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345bc16a50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534590d8600>, 9884069.834165294)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345d55f450>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534590db150>, 9884070.1881329)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345bc16710>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345bc25510>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534583efcd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345d581a50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441c5c8a0>, 9884071.045061488)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345f66e6d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345941f770>, 9884068.507405704)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345d580e50>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534590ca270>, 9884070.284294892)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345bc27a10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345941c7c0>, 9884067.91974915)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534583ee710>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345a265f90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534591f3380>, 9884071.457695032)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345a265dd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345a4cf350>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534585172d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a5f2890>, 9884072.390906492)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345a751050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534590ca900>, 9884071.29269405)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458514ed0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458630bd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458fc2c80>, 9884073.57149553)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458633c50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345869a350>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458fc0600>, 9884071.838181453)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345869ad90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458800a10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458fc1240>, 9884077.442622684)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534588011d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153443f990d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534590da120>, 9884072.968859527)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345bc244d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534430b0a90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458924890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458c9f690>, 9884072.896125732)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534430b22d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458c9fb60>, 9884072.419690756)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534589254d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345a112290>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345a2f38d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153443061e90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345901f150>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153443057710>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345a073a10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a5f1e10>, 9884070.65575398)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345a072310>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534590c98d0>, 9884070.691495717)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345901e910>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534590c9e80>, 9884072.874300389)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345a110f90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534590d9a20>, 9884070.48979056)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345d3ceb10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534590c87c0>, 9884070.59028738)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153443060890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458fc3a80>, 9884070.957655732)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153443085b90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345a158f90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458c9cd70>, 9884074.790964426)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345a159610>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534585a3210>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458c9ca60>, 9884074.29446544)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345d60fe10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534585334d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458c9d470>, 9884074.882953843)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442ebe950>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441846950>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458c9f380>, 9884074.734507976)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441846810>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458f06510>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534419d6c10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441b55630>, 9884074.116525756)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458ab3750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441b54e50>, 9884076.1194138)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534419d6cd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441890490>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441b56190>, 9884073.608432952)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441890350>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441701650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441b542f0>, 9884073.678197479)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441701610>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534431f7d90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441b56820>, 9884078.000853239)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534431f6290>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344195d990>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441b54ec0>, 9884074.395300036)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344195dd50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344194a190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441b54f30>, 9884075.970418848)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441733190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441915bd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441b54210>, 9884074.867253903)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441915710>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441aed9d0>\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metricsmetrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      " {'f1': 0.8, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.28571428571428575, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.18181818181818182, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.4, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.2857142857142857, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.2857142857142857, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534419035d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345bbc6f90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442ed8910>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534431091d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534590c8de0>, 9884073.244033743)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442ed9ed0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345941d860>, 9884073.289432555)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534430570d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441b56ba0>, 9884075.382321933)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441aed650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441b54600>, 9884075.202725036)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534586c83d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458c9c2f0>, 9884072.817978663)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345bbc4d10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441973c50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458316c10>, 9884075.26013316)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441886b90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534418f1cd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441b55400>, 9884076.127162669)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534418f3790>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534418e87d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458fc0e50>, 9884077.260308104)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534418ea4d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534419a0e10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441aba050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458317070>, 9884076.845395574)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534419a0d50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458315400>, 9884078.21851227)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441ab9410>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534418b7a10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458317460>, 9884076.337204237)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534418b4710>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441a75210>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458314fa0>, 9884077.964436913)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441a76690>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441a484d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345896e9e0>, 9884076.513830708)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441949650>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1533fad2d450>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458c9fe00>, 9884076.791536875)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441bc8350>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441aa0b90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441b96210>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441b569e0>, 9884076.599892313)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441a59710>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a5f1320>, 9884076.88098704)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441b96ad0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441bf3ed0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441b5b010>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345896fe70>, 9884081.31024997)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345a073610>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345896ec80>, 9884076.998539226)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441b5a910>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441cef3d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441cc7490>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534419e8790>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345896f380>, 9884080.86853824)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441ceefd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458317e00>, 9884076.007909872)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534419ea2d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534590d8f30>, 9884077.485643491)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441cc6f90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345a4ce110>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458fc2890>, 9884079.087167984)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345f66e950>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344171bd50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345896d710>, 9884082.081866875)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441718550>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441def2d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458a70590>, 9884078.587863417)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441dedfd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441ce46d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458316ac0>, 9884079.504368559)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441a33190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441d9e7d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441d59990>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458a71940>, 9884080.994349632)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441bafa10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458a73380>, 9884079.718769047)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441c50410>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441c284d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345a246690>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345896c0c0>, 9884081.528772805)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441c29cd0>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343fd6b7e0>, 9884078.940092811)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345a244210>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441d03290>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441d77650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458536d60>, 9884080.884531453)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441d77b90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345ad24190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441e76990>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458fc1ef0>, 9884080.04753352)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441abafd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534585360b0>, 9884080.644364253)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441e76090>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441c354d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441b1a210>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441b7a250>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458a73930>, 9884093.930982368)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441ec9990>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458a70360>, 9884078.098604882)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441c37b90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458a708a0>, 9884078.256475592)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441b19190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458a73bd0>, 9884078.456202816)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441b794d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441d13390>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534585356a0>, 9884081.408171214)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441d13d10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441f98a90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458437070>, 9884081.855125688)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441f98f90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441f6f5d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441f70190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458536660>, 9884080.704204302)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441dff9d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458436f20>, 9884081.459424617)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441f72910>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441c1e250>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345896f310>, 9884083.025561055)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441f885d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534420e08d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458437310>, 9884081.332871504)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534420e0190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441a30fd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458435f60>, 9884082.851332562)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441bafa90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442089890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534584367b0>, 9884083.140028048)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534420893d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441fc5d10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458534ad0>, 9884082.718906965)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441fb4e50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442000350>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534585357f0>, 9884083.067270985)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442003750>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534421df7d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441e34950>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441b97050>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441fe74d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441feecd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458a70e50>, 9884080.731191382)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441fef950>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441b56b30>, 9884080.99312478)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441d329d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345896ea50>, 9884083.532136474)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534421de050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458fc14e0>, 9884080.123141188)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441dca6d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458536c10>, 9884080.383900004)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441fe40d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441f29050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458437150>, 9884087.603528876)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441f28350>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344216add0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459f8f3f0>, 9884085.144748284)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344216aad0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441ea31d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459f8d710>, 9884084.6732642)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441ea3ad0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534421c6f10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459f8d550>, 9884084.237788664)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534421c5b50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441ab9750>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344227afd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534583165f0>, 9884084.154294042)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441ab9e90>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442263110>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459f8e0b0>, 9884085.342079721)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442261f10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442292150>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442217a10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459f8e190>, 9884089.13656738)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442292cd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459f8c1a0>, 9884085.114253452)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534422171d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442203790>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459513460>, 9884084.155545335)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442201b10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441a06ad0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459511710>, 9884084.28054042)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344206b190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344238a610>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344234bb10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459f8f700>, 9884085.347116677)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344234bfd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534423ec3d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458437620>, 9884089.094607612)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442313750>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441a8bf50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459f8c130>, 9884085.382828906)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534422799d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458536430>, 9884085.067826852)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344238b490>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459f8f690>, 9884083.420889292)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534421685d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459f8cc20>, 9884086.864361422)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534423ecf90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344217acd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459f8db00>, 9884083.000791537)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442179d10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442307850>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459511d30>, 9884087.412329324)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442305350>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345d437a90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459512510>, 9884085.7575324)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344207e910>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344229ed90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442457e90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459513540>, 9884085.821586542)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442131e10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459510d00>, 9884087.705763808)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442455b50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534424b4ed0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442401210>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345962eeb0>, 9884086.157210376)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534424b5e90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345962c8a0>, 9884087.875974577)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442402f10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534425bddd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345962c0c0>, 9884087.070740504)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344247aed0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534425544d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345962f700>, 9884088.085287644)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442159c10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442255550>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345962d240>, 9884088.009878604)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442254350>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442536b90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345962c3d0>, 9884087.854804972)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442534d10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344257de50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534426c1250>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345962f620>, 9884088.669555556)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442132ad0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345962e120>, 9884088.197611134)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534426c3ad0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344234be50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344246b650>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441a04b50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534584373f0>, 9884085.777533334)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441cec290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459738050>, 9884089.345066644)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344234bc90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345962c590>, 9884086.319114449)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344200f110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442e8f350>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459738280>, 9884089.768869055)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442e8de90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442569690>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534597394e0>, 9884089.677076038)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442568190>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534426f0790>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534426dd150>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534425d0a50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459510210>, 9884090.319607897)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534425d0c90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344274e790>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458434c20>, 9884090.063808246)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344274eb50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344234a710>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534427461d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345993d550>, 9884089.959812108)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442d3dcd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345993eac0>, 9884091.05235393)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442744150>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534427165d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459510440>, 9884091.67562304)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442716a50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442cd0bd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534427384d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345993da20>, 9884094.388244988)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442cd1350>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345993f230>, 9884090.390178343)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441dfdf10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344241dfd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442fd6c90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534425850d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442638050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534597391d0>, 9884089.3016037)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534426f3390>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345973a820>, 9884091.14699885)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534427d8b90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459738c90>, 9884088.760439098)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442585590>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345962de80>, 9884091.151225723)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344241c750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459738ec0>, 9884088.518487062)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442fd6e50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459738b40>, 9884089.0898536)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344263a950>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344279ff50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442890dd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345993f380>, 9884091.307670947)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344279d290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534597396a0>, 9884092.033698944)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442891310>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344231e150>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458534c20>, 9884092.22613086)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344231f110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442313390>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459c5db00>, 9884091.429306045)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442313010>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534427f6ad0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459513700>, 9884093.300096257)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534427f4610>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534427a2410>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459739be0>, 9884092.507392982)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534427a3e50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442fb5990>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534429caa50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459c5f0e0>, 9884091.965511614)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442e8ecd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459c5d6a0>, 9884092.58428779)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534429ca810>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534429f5750>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534429e2890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345993fa10>, 9884091.716885071)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534429f4650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459c5f8c0>, 9884093.228618275)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534429e0c10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442e069d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345962c050>, 9884092.674236916)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442e06d90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442971b10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534428bb010>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442d9b7d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534429e4e10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345993c130>, 9884091.390826782)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534428b8310>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343fd6ae40>, 9884094.296845954)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442972910>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345962cd70>, 9884092.09835106)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442d99310>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459c5de10>, 9884091.781263204)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534429e6c10>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442790a50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a199a20>, 9884093.967874605)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442984190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442e25d90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a1997f0>, 9884093.777137464)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442e25ed0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534428a4bd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442924310>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a19acf0>, 9884096.045862537)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534428a6550>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a19a0b0>, 9884095.26705018)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534429245d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344294b010>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a4c8d00>, 9884094.011301436)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442948ed0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534429ca410>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344097d910>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459c5edd0>, 9884094.693633456)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534429cb450>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a4cbf50>, 9884095.08191204)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344097f210>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345d36ec90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a4ca5f0>, 9884096.276096076)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345d36e150>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534407e16d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a4c9b00>, 9884094.311659576)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442d30e90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f552490>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f56c290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459513ee0>, 9884094.463914612)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f552210>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a4ca430>, 9884094.585096028)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f563a10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f571b10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a1992b0>, 9884095.34088196)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f5719d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442eee3d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442e941d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344296b490>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a19a3c0>, 9884093.348844511)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442e96dd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a198520>, 9884093.31757886)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442c95e10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a1986e0>, 9884093.197183538)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442968ed0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f597890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a4c9010>, 9884095.335794356)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f597ad0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f583490>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f5c4a90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459c5db70>, 9884096.943103852)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534428aefd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459e771c0>, 9884097.952508094)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f5bb190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442dd1f10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a4cb3f0>, 9884096.915235309)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442dd0250>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f5521d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f434e10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459c5cc90>, 9884097.2712121)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f551e10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a19ac80>, 9884096.07741938)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f4349d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f43e0d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459c5c830>, 9884096.953265484)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f43ee50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442972690>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f461010>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459e77e00>, 9884097.251668079)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534429722d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a4c91d0>, 9884096.468666447)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f461750>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f470890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459e77d20>, 9884096.63756074)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f470690>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f414bd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459e758d0>, 9884097.091921052)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442c3c110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f492750>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f4ada90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459e74830>, 9884097.158482453)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f4929d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f4c0050>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442e26ad0>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f5cf150>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a4cb4d0>, 9884096.977706289)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f4adf50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459c5f310>, 9884098.00710833)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f4c0350>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a4cab30>, 9884095.1176043)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442e24990>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459e74bb0>, 9884095.217885492)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f5cef50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f572d10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a3b7000>, 9884098.158755379)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f572dd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f4f3850>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f30a410>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a3b5b70>, 9884097.36732457)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f4e5010>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a3b7ee0>, 9884097.916124737)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f30a210>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f335350>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f33e6d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459e77850>, 9884098.503253313)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f33c0d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f34ba10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a19bf50>, 9884098.652239995)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f34bf50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f37c050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458d9e5f0>, 9884099.340738563)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f37c550>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f39c710>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f2fef90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f3bf490>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a3b7460>, 9884099.721218916)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f4361d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a4ca4a0>, 9884099.76913121)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f3bf350>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f3c6d90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458d9cde0>, 9884099.796274064)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f3c6c50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f4e1a50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459c5cc20>, 9884100.07934263)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f1fc790>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f4e0c50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f5cfa50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458d9e120>, 9884100.59686302)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f335150>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a3b7e70>, 9884099.499721268)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f39c5d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a3b5c50>, 9884097.690682497)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f4e0a50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458d9c6e0>, 9884099.21848748)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f3d37d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a3b60b0>, 9884097.481451828)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f5cf690>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f3b0910>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f2204d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441d70d00>, 9884102.758419923)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f3b0990>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459c5e890>, 9884100.262377094)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f220b90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f238210>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459e76e40>, 9884102.523688888)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f238710>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f244d10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441d707c0>, 9884100.91845899)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f244b10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f2574d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f26ef50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458d9fd90>, 9884102.70165354)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f257b10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a19b380>, 9884100.001659237)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f26ed50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f3e0390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f2904d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441d709f0>, 9884103.444979075)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f3b31d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f296450>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441d73a80>, 9884101.451603804)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f296310>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f2ad850>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441d71b70>, 9884101.11105798)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f2ad710>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f2afa10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a4c8d70>, 9884101.712791933)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f2afd50>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f2edf90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f0fb710>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459e75550>, 9884101.748441048)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f0fb5d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f11ca50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f2068d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a3b4520>, 9884100.386589617)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f290bd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458d9c9f0>, 9884101.760667011)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f2ee710>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441f8f070>, 9884101.455887971)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f0fba90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441d72040>, 9884099.743705776)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f206e50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f13fd10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f143810>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441f8d940>, 9884103.074276512)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f13fbd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441f8d780>, 9884102.751491113)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f1436d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f2b9990>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441f8e200>, 9884101.796979228)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f207650>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f128c10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f17c450>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441f8f310>, 9884114.114075424)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f4444d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441f8f380>, 9884103.215965897)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f177b90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f101fd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534421a0980>, 9884102.737137623)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f101cd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f168c90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f1ae1d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534421a1f60>, 9884104.08220599)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f196610>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534421a1240>, 9884103.226316843)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f1ae890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f1d5210>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f11c5d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441f8f5b0>, 9884103.758206405)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f11c410>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f2ef6d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f1e1b90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534421a0fa0>, 9884103.767166227)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f2ee510>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534421a31c0>, 9884103.902799746)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f1d7990>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f1bf110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f031a10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534421a0670>, 9884104.537429707)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f1bfd90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441f8cbb0>, 9884104.354817787)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f479f90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f03d190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441f8dfd0>, 9884104.193024091)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f047510>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f26ef10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441f8f000>, 9884105.210636022)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f1d4e50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a4c8910>, 9884104.285589669)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f03d910>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441f8e430>, 9884101.397942036)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f26eb50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f05cdd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534421a1d30>, 9884104.957881855)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f053190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f09d4d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441f8c2f0>, 9884105.644007623)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f09e690>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f3b3290>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f0d0f90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441e6ff50>, 9884105.5832572)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f3b2d90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441e6e740>, 9884105.031790173)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f0d0b50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f079110>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441e6cc90>, 9884105.997831784)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f079250>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f14c950>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441e6fe70>, 9884105.166705614)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f14ccd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343eeff7d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343ef14390>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441e6d9b0>, 9884106.095233032)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343eeffa10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441e6c0c0>, 9884105.390731508)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343ef14a50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343ef2c1d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441e6f2a0>, 9884105.708976595)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343ef2c6d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343ef4c3d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534421a1160>, 9884106.197479373)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343ef39c10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343eefe110>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534423c5e80>, 9884106.271943973)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343eefeed0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f1be110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343ef74750>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f3e06d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534423c7a80>, 9884107.217103297)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343ef61390>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534423c5390>, 9884107.442606805)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343ef74dd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441f8d860>, 9884104.564323476)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f18d610>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343ef7fd50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534423c52b0>, 9884108.310752615)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343ef7fc10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343ef31c50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534423c7b60>, 9884109.059097562)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343ef31990>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343ef9c8d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343efb7c50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534423c4440>, 9884106.81373936)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343ef9fcd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534423c62e0>, 9884107.499061607)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343efb7b10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343ef7d4d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534423c74d0>, 9884107.744180659)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343efbda50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343ebfb250>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534423c54e0>, 9884109.294226073)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343ebfb110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f244d90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441e6e200>, 9884121.458035177)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f1ae190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343ec16150>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343ec25050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153443ed5a20>, 9884107.601272443)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343ec164d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153443ed7690>, 9884107.460709153)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343ec25710>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343efe1ed0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153443ed7d20>, 9884107.375734512)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343efe1fd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343ec149d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153443ed5cc0>, 9884108.698506117)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343ec15210>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f05f350>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343ec79650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441e6f310>, 9884108.39952349)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f19ee50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153443ed6430>, 9884108.176443731)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343ec79d10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343ec3bc50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153443ed4b40>, 9884108.473888302)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343ec3bc90>\n",
      "Evaluating workflow:  24%|██▍       | 122/500 [00:21<00:53,  7.02it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  25%|██▍       | 124/500 [00:22<01:29,  4.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.7777777777777778, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  25%|██▌       | 127/500 [00:23<01:13,  5.11it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  26%|██▌       | 130/500 [00:23<01:08,  5.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  26%|██▋       | 132/500 [00:23<00:51,  7.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.2857142857142857, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  27%|██▋       | 134/500 [00:24<00:49,  7.36it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  27%|██▋       | 136/500 [00:24<00:50,  7.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  28%|██▊       | 139/500 [00:24<00:48,  7.46it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  29%|██▊       | 143/500 [00:25<00:47,  7.58it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.923076923076923, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  29%|██▉       | 144/500 [00:25<00:52,  6.78it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  29%|██▉       | 145/500 [00:25<01:13,  4.84it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.039999999999999994, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  30%|██▉       | 149/500 [00:26<00:45,  7.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.42857142857142855, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  30%|███       | 151/500 [00:26<00:43,  8.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.6, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  31%|███       | 154/500 [00:26<00:49,  6.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  31%|███▏      | 157/500 [00:27<01:02,  5.48it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.2666666666666667, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  32%|███▏      | 159/500 [00:28<01:08,  5.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  33%|███▎      | 164/500 [00:28<00:39,  8.61it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.14285714285714288, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  33%|███▎      | 166/500 [00:28<00:34,  9.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  34%|███▎      | 168/500 [00:29<00:49,  6.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  34%|███▍      | 170/500 [00:29<00:46,  7.17it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.625, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  34%|███▍      | 172/500 [00:30<01:36,  3.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.1904761904761905, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  35%|███▍      | 173/500 [00:31<01:29,  3.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  35%|███▌      | 177/500 [00:31<00:58,  5.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  37%|███▋      | 183/500 [00:31<00:35,  8.87it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.5, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  37%|███▋      | 185/500 [00:32<00:46,  6.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.09523809523809525, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.23076923076923078, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  37%|███▋      | 187/500 [00:32<00:43,  7.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.28571428571428575, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  38%|███▊      | 189/500 [00:32<00:42,  7.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.3076923076923077, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  38%|███▊      | 190/500 [00:33<00:47,  6.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  38%|███▊      | 192/500 [00:33<00:56,  5.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  39%|███▊      | 193/500 [00:33<00:53,  5.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  39%|███▉      | 194/500 [00:34<01:21,  3.77it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  40%|███▉      | 199/500 [00:34<00:50,  5.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.5, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  41%|████      | 203/500 [00:35<00:40,  7.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.4, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.7777777777777778, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  41%|████▏     | 207/500 [00:35<00:25, 11.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.19999999999999998, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  42%|████▏     | 211/500 [00:36<00:42,  6.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.4, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  43%|████▎     | 215/500 [00:37<00:39,  7.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.8, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  43%|████▎     | 217/500 [00:37<01:00,  4.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  44%|████▎     | 218/500 [00:38<01:09,  4.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.4, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  44%|████▍     | 221/500 [00:38<00:50,  5.58it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  44%|████▍     | 222/500 [00:38<00:54,  5.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.4, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  45%|████▍     | 223/500 [00:39<01:00,  4.57it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.21428571428571425, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.6153846153846153, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  46%|████▌     | 228/500 [00:39<00:42,  6.46it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.15384615384615385, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  46%|████▌     | 231/500 [00:39<00:32,  8.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  47%|████▋     | 235/500 [00:40<00:27,  9.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.5714285714285715, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.5, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  47%|████▋     | 237/500 [00:40<00:34,  7.66it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  48%|████▊     | 238/500 [00:41<00:56,  4.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  48%|████▊     | 239/500 [00:41<01:14,  3.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  48%|████▊     | 240/500 [00:42<01:11,  3.66it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  48%|████▊     | 242/500 [00:42<00:58,  4.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  49%|████▉     | 247/500 [00:42<00:31,  8.09it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  50%|████▉     | 248/500 [00:43<00:33,  7.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  50%|█████     | 251/500 [00:43<00:26,  9.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.2666666666666667, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  51%|█████     | 253/500 [00:43<00:43,  5.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.4444444444444445, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  51%|█████     | 256/500 [00:44<00:36,  6.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.5454545454545454, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  51%|█████▏    | 257/500 [00:44<00:42,  5.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  52%|█████▏    | 260/500 [00:44<00:34,  6.86it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.13333333333333336, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.5714285714285715, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  53%|█████▎    | 263/500 [00:45<00:32,  7.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.8, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  53%|█████▎    | 265/500 [00:45<00:38,  6.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.8571428571428571, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.7499999999999999, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  53%|█████▎    | 267/500 [00:46<00:38,  6.02it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.4, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  54%|█████▍    | 271/500 [00:46<00:33,  6.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  55%|█████▍    | 273/500 [00:46<00:29,  7.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.22222222222222224, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  55%|█████▌    | 276/500 [00:47<00:25,  8.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  56%|█████▌    | 280/500 [00:47<00:25,  8.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.5142857142857142, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  57%|█████▋    | 283/500 [00:48<00:33,  6.48it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  57%|█████▋    | 285/500 [00:48<00:28,  7.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.2857142857142857, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  57%|█████▋    | 286/500 [00:48<00:37,  5.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  58%|█████▊    | 290/500 [00:49<00:28,  7.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.3076923076923077, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  58%|█████▊    | 292/500 [00:49<00:39,  5.27it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.8, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  59%|█████▉    | 295/500 [00:50<00:29,  7.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.3333333333333333, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.23529411764705882, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  60%|█████▉    | 298/500 [00:50<00:24,  8.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.9090909090909091, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.012903225806451615, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  60%|██████    | 300/500 [00:50<00:28,  7.11it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.5, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  61%|██████    | 304/500 [00:51<00:25,  7.58it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.4, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  61%|██████    | 305/500 [00:51<00:27,  6.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.4444444444444445, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  62%|██████▏   | 308/500 [00:52<00:28,  6.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  62%|██████▏   | 311/500 [00:53<00:48,  3.86it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  63%|██████▎   | 316/500 [00:53<00:28,  6.37it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.3571428571428571, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.7058823529411764, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  64%|██████▍   | 322/500 [00:54<00:24,  7.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.5714285714285715, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  65%|██████▌   | 326/500 [00:55<00:23,  7.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.8, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.8, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  66%|██████▌   | 329/500 [00:55<00:22,  7.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.5714285714285715, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  66%|██████▌   | 330/500 [00:55<00:30,  5.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  67%|██████▋   | 334/500 [00:56<00:23,  7.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  67%|██████▋   | 336/500 [00:56<00:18,  8.71it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.3157894736842105, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  68%|██████▊   | 340/500 [00:57<00:22,  7.16it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  69%|██████▉   | 344/500 [00:57<00:15,  9.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.8, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  70%|██████▉   | 348/500 [00:58<00:29,  5.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.3333333333333333, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.4, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  70%|███████   | 350/500 [00:58<00:23,  6.28it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  71%|███████   | 355/500 [00:59<00:15,  9.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.33333333333333337, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  71%|███████▏  | 357/500 [00:59<00:15,  9.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.047619047619047616, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.8, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  72%|███████▏  | 359/500 [00:59<00:18,  7.71it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  72%|███████▏  | 361/500 [01:00<00:19,  7.07it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.8, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  73%|███████▎  | 363/500 [01:00<00:27,  5.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.5714285714285715, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.21052631578947367, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  73%|███████▎  | 366/500 [01:01<00:21,  6.11it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  74%|███████▍  | 369/500 [01:01<00:21,  6.06it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.8, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  74%|███████▍  | 371/500 [01:01<00:15,  8.07it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.6363636363636364, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.75, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  75%|███████▍  | 374/500 [01:02<00:19,  6.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  75%|███████▌  | 376/500 [01:02<00:18,  6.67it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  76%|███████▌  | 379/500 [01:03<00:21,  5.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.4444444444444445, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  77%|███████▋  | 384/500 [01:04<00:14,  8.16it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.10526315789473684, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  77%|███████▋  | 386/500 [01:04<00:15,  7.46it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.4444444444444445, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  77%|███████▋  | 387/500 [01:04<00:15,  7.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.26666666666666666, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  78%|███████▊  | 389/500 [01:04<00:17,  6.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.6, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.5, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  78%|███████▊  | 391/500 [01:05<00:21,  5.05it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  79%|███████▉  | 395/500 [01:05<00:12,  8.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  79%|███████▉  | 397/500 [01:06<00:15,  6.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  80%|███████▉  | 398/500 [01:06<00:20,  4.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.058823529411764705, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  80%|████████  | 401/500 [01:07<00:17,  5.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.3333333333333333, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  81%|████████  | 403/500 [01:07<00:16,  5.72it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.5454545454545454, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  81%|████████  | 406/500 [01:07<00:12,  7.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.5, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.8, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  82%|████████▏ | 410/500 [01:08<00:11,  7.84it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.18181818181818182, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.5, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.8, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  82%|████████▏ | 411/500 [01:08<00:11,  8.07it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  82%|████████▏ | 412/500 [01:08<00:17,  4.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.3, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  83%|████████▎ | 414/500 [01:09<00:20,  4.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  83%|████████▎ | 417/500 [01:09<00:15,  5.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.4, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  84%|████████▍ | 420/500 [01:10<00:13,  6.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  85%|████████▍ | 423/500 [01:10<00:10,  7.55it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.5, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  85%|████████▌ | 425/500 [01:10<00:07,  9.46it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  85%|████████▌ | 427/500 [01:11<00:08,  8.57it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.2857142857142857, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  86%|████████▌ | 430/500 [01:11<00:09,  7.61it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  86%|████████▌ | 431/500 [01:11<00:11,  6.11it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  86%|████████▋ | 432/500 [01:12<00:14,  4.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.5, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  87%|████████▋ | 434/500 [01:12<00:11,  5.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  87%|████████▋ | 435/500 [01:12<00:13,  4.89it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.4210526315789474, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.4, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  88%|████████▊ | 438/500 [01:13<00:09,  6.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  88%|████████▊ | 441/500 [01:13<00:08,  6.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.8235294117647058, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  88%|████████▊ | 442/500 [01:13<00:11,  5.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.4444444444444445, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  89%|████████▊ | 443/500 [01:14<00:12,  4.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  89%|████████▉ | 447/500 [01:14<00:09,  5.36it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.22222222222222218, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  90%|█████████ | 450/500 [01:14<00:06,  7.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.5454545454545454, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  90%|█████████ | 452/500 [01:15<00:06,  7.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  91%|█████████ | 453/500 [01:15<00:09,  5.11it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.5, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  91%|█████████ | 454/500 [01:16<00:10,  4.29it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.19999999999999998, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.6153846153846153, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  91%|█████████▏| 457/500 [01:16<00:07,  5.84it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  92%|█████████▏| 458/500 [01:16<00:07,  5.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.2857142857142857, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  92%|█████████▏| 461/500 [01:17<00:06,  6.07it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.47619047619047616, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  92%|█████████▏| 462/500 [01:17<00:07,  5.36it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  93%|█████████▎| 466/500 [01:17<00:04,  6.88it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.47619047619047616, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  94%|█████████▎| 468/500 [01:18<00:05,  5.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  94%|█████████▍| 472/500 [01:18<00:03,  7.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.4, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  95%|█████████▍| 474/500 [01:18<00:03,  8.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.4444444444444445, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  95%|█████████▌| 475/500 [01:19<00:06,  3.72it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.2857142857142857, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  96%|█████████▌| 481/500 [01:20<00:03,  5.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.5714285714285715, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  96%|█████████▋| 482/500 [01:20<00:03,  5.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  97%|█████████▋| 484/500 [01:21<00:03,  5.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  97%|█████████▋| 486/500 [01:21<00:02,  6.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.33333333333333337, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.8, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.3333333333333333, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  98%|█████████▊| 488/500 [01:21<00:02,  5.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.10526315789473684, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  98%|█████████▊| 490/500 [01:22<00:01,  5.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.8, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  99%|█████████▉| 494/500 [01:22<00:00,  6.04it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.5, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  99%|█████████▉| 496/500 [01:23<00:00,  4.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.33333333333333337, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow: 100%|█████████▉| 498/500 [01:24<00:00,  3.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.375, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow: 100%|█████████▉| 499/500 [01:26<00:00,  1.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow: 100%|██████████| 500/500 [01:31<00:00,  5.49it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.04347826086956522, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   1%|          | 5/500 [00:02<02:48,  2.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.75, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.3333333333333333, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.18181818181818182, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   2%|▏         | 11/500 [00:02<01:09,  7.05it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.4, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   3%|▎         | 16/500 [00:03<01:05,  7.40it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.5454545454545454, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   4%|▍         | 20/500 [00:04<01:27,  5.47it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   5%|▍         | 23/500 [00:04<01:11,  6.66it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   5%|▌         | 25/500 [00:05<01:27,  5.40it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   6%|▌         | 28/500 [00:05<01:16,  6.17it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   6%|▌         | 29/500 [00:05<01:25,  5.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.7272727272727273, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   6%|▌         | 30/500 [00:06<01:56,  4.04it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   7%|▋         | 33/500 [00:06<01:36,  4.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   7%|▋         | 34/500 [00:07<02:25,  3.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.4444444444444445, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   7%|▋         | 36/500 [00:08<02:03,  3.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.3333333333333333, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   8%|▊         | 40/500 [00:08<01:01,  7.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.1904761904761905, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.17391304347826086, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   8%|▊         | 42/500 [00:08<01:06,  6.86it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   9%|▉         | 44/500 [00:09<01:27,  5.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   9%|▉         | 46/500 [00:09<01:24,  5.40it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.5454545454545454, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  10%|█         | 51/500 [00:10<00:58,  7.66it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  11%|█         | 53/500 [00:10<01:01,  7.32it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.1739130434782609, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  11%|█         | 54/500 [00:10<01:42,  4.36it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  11%|█▏        | 57/500 [00:11<01:21,  5.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  12%|█▏        | 59/500 [00:11<01:17,  5.72it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.125, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.5, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  12%|█▏        | 62/500 [00:12<01:34,  4.63it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  13%|█▎        | 65/500 [00:12<01:01,  7.09it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  13%|█▎        | 67/500 [00:12<00:51,  8.42it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  14%|█▍        | 69/500 [00:13<01:01,  7.05it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.33333333333333337, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  14%|█▍        | 71/500 [00:13<01:03,  6.77it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.30769230769230765, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  15%|█▍        | 73/500 [00:13<01:03,  6.71it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  15%|█▍        | 74/500 [00:14<01:09,  6.11it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.8, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  15%|█▌        | 75/500 [00:14<01:18,  5.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  15%|█▌        | 77/500 [00:14<01:26,  4.92it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  16%|█▌        | 79/500 [00:15<01:17,  5.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  16%|█▌        | 80/500 [00:15<01:11,  5.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  16%|█▋        | 82/500 [00:15<00:58,  7.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  17%|█▋        | 84/500 [00:16<01:21,  5.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  17%|█▋        | 85/500 [00:16<01:28,  4.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  17%|█▋        | 87/500 [00:16<01:29,  4.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.10526315789473684, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  18%|█▊        | 90/500 [00:17<01:24,  4.83it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  18%|█▊        | 91/500 [00:17<01:21,  5.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  18%|█▊        | 92/500 [00:17<01:28,  4.61it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.6, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  19%|█▉        | 96/500 [00:18<00:58,  6.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  20%|█▉        | 98/500 [00:18<00:46,  8.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.28571428571428575, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  20%|██        | 100/500 [00:18<00:54,  7.28it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.28571428571428575, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  20%|██        | 102/500 [00:19<01:35,  4.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.4, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  21%|██        | 103/500 [00:19<01:22,  4.82it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  21%|██        | 105/500 [00:20<01:08,  5.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.1904761904761905, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  21%|██▏       | 107/500 [00:20<00:59,  6.58it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.19999999999999998, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.4615384615384615, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  22%|██▏       | 111/500 [00:20<00:50,  7.77it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  23%|██▎       | 114/500 [00:21<00:42,  9.14it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.5, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.3076923076923077, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.8, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.16666666666666669, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  24%|██▍       | 119/500 [00:21<00:28, 13.46it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.4615384615384615, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  24%|██▍       | 121/500 [00:21<00:46,  8.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.9090909090909091, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  25%|██▌       | 126/500 [00:22<00:54,  6.87it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.2857142857142857, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.7368421052631579, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  26%|██▌       | 128/500 [00:23<00:53,  6.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  26%|██▌       | 130/500 [00:23<01:08,  5.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  27%|██▋       | 136/500 [00:24<00:51,  7.09it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.21052631578947367, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.2857142857142857, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  28%|██▊       | 138/500 [00:24<00:43,  8.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.4, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  28%|██▊       | 140/500 [00:25<01:09,  5.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  29%|██▊       | 143/500 [00:25<01:04,  5.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.6, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.21052631578947367, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  30%|██▉       | 148/500 [00:26<00:36,  9.71it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  30%|███       | 150/500 [00:26<00:52,  6.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  30%|███       | 152/500 [00:26<00:43,  7.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.7692307692307693, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  31%|███       | 154/500 [00:26<00:44,  7.71it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.33333333333333337, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  31%|███       | 156/500 [00:28<01:44,  3.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  32%|███▏      | 158/500 [00:28<01:25,  3.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  32%|███▏      | 162/500 [00:29<01:05,  5.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.5, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.17391304347826084, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  33%|███▎      | 166/500 [00:29<00:45,  7.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.7142857142857143, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  33%|███▎      | 167/500 [00:29<00:46,  7.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  34%|███▎      | 168/500 [00:30<01:50,  3.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  34%|███▍      | 171/500 [00:31<01:27,  3.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  35%|███▌      | 175/500 [00:31<01:01,  5.28it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.5, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  35%|███▌      | 177/500 [00:32<00:50,  6.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.42857142857142855, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  36%|███▌      | 179/500 [00:32<00:55,  5.82it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.08333333333333334, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  36%|███▌      | 180/500 [00:32<01:12,  4.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  36%|███▌      | 181/500 [00:33<01:19,  4.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.3076923076923077, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  36%|███▋      | 182/500 [00:33<01:40,  3.17it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  37%|███▋      | 185/500 [00:34<01:12,  4.35it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.5, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  38%|███▊      | 188/500 [00:34<00:57,  5.47it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  38%|███▊      | 189/500 [00:35<01:03,  4.88it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.4, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  38%|███▊      | 192/500 [00:35<00:57,  5.35it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.07407407407407408, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.5, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  39%|███▊      | 193/500 [00:36<01:22,  3.74it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  39%|███▉      | 194/500 [00:36<01:30,  3.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  39%|███▉      | 196/500 [00:36<01:09,  4.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  40%|███▉      | 198/500 [00:37<01:02,  4.82it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  40%|████      | 200/500 [00:37<01:02,  4.77it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.0625, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  40%|████      | 202/500 [00:37<00:49,  6.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.19999999999999998, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  41%|████      | 203/500 [00:38<00:47,  6.28it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  41%|████      | 206/500 [00:38<00:40,  7.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.14285714285714288, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  42%|████▏     | 208/500 [00:38<00:38,  7.61it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.2857142857142857, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  42%|████▏     | 210/500 [00:39<00:54,  5.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.4, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  42%|████▏     | 211/500 [00:39<01:08,  4.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.8, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  43%|████▎     | 214/500 [00:40<00:52,  5.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.7777777777777778, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  43%|████▎     | 217/500 [00:40<00:34,  8.09it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.18181818181818182, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.4, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  44%|████▍     | 220/500 [00:40<00:47,  5.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  44%|████▍     | 221/500 [00:41<00:53,  5.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  44%|████▍     | 222/500 [00:41<00:58,  4.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.15384615384615385, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.6153846153846153, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.21428571428571425, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  45%|████▌     | 226/500 [00:41<00:38,  7.16it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  46%|████▌     | 228/500 [00:42<00:53,  5.09it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  46%|████▌     | 229/500 [00:43<01:23,  3.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  47%|████▋     | 234/500 [00:43<00:42,  6.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  48%|████▊     | 238/500 [00:44<00:35,  7.38it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.5, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  48%|████▊     | 241/500 [00:44<00:29,  8.86it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.14285714285714288, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  49%|████▉     | 244/500 [00:45<00:55,  4.61it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  49%|████▉     | 247/500 [00:45<00:44,  5.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  50%|████▉     | 249/500 [00:46<00:38,  6.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.14285714285714285, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.2857142857142857, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  50%|█████     | 252/500 [00:46<00:33,  7.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  51%|█████     | 253/500 [00:46<00:36,  6.83it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.4444444444444445, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  51%|█████     | 255/500 [00:47<00:35,  6.96it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  52%|█████▏    | 258/500 [00:47<00:42,  5.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.1, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  52%|█████▏    | 259/500 [00:47<00:38,  6.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.5454545454545454, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  52%|█████▏    | 261/500 [00:48<00:40,  5.92it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  53%|█████▎    | 263/500 [00:48<00:44,  5.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  53%|█████▎    | 265/500 [00:48<00:35,  6.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.5, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  53%|█████▎    | 266/500 [00:48<00:33,  6.96it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  53%|█████▎    | 267/500 [00:49<00:39,  5.84it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.33333333333333337, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  54%|█████▍    | 269/500 [00:49<00:43,  5.36it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  54%|█████▍    | 270/500 [00:49<00:49,  4.63it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.8, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f01ec90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534424c7d90>, 9884109.920343217)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343ef61290>\n",
      "Evaluating workflow:  54%|█████▍    | 271/500 [00:50<01:11,  3.20it/s]Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343eb73490>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1533fadeeac0>, 9884111.623940656)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343eb73f90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343eb66350>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1533faded2b0>, 9884112.407826945)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343ec48250>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343eb73ad0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1533fadeec80>, 9884114.091456752)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343ef31c90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343ea23c50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a8027b0>, 9884115.66858478)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343ea23f50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343ea65fd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a8020b0>, 9884113.203652415)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343ea655d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343e9a2910>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a6f1be0>, 9884114.563686844)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343e9a2c90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343eafbd90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a6f2270>, 9884116.524652308)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343e9b5790>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343e9b0150>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a4df620>, 9884115.146816468)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343e9b0910>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343e928f90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343eafbf50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343ea22d10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343e9bf2d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a6f1860>, 9884114.856065264)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f161c10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a6f2350>, 9884113.75427996)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343e92a490>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a6f0fa0>, 9884115.218593892)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343ea228d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a4df380>, 9884115.25268576)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343e9bf0d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343e815bd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a801550>, 9884119.00114484)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343e816890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343e716410>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441d73310>, 9884117.767140549)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343e716750>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343e8b6110>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a090e50>, 9884119.710059203)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343ec5ed90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343e709d10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a9322e0>, 9884119.615831794)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343e709610>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343e856e90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343e95b150>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a4deac0>, 9884117.162096111)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343e856bd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a091780>, 9884117.093401572)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343e95b210>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343eca9410>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343ef6dd10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f49be10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f4e42d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345a3a2010>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442783190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343e7ac0d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343e7af510>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343e7da210>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345863c210>, 9884130.993713852)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343eca9990>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15344008a970>, 9884131.678294372)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343ef6cdd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345896fee0>, 9884131.287139831)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344273be10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a4dd550>, 9884119.617958903)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343e7ac510>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a932740>, 9884118.998961236)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343e7af3d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a4de510>, 9884128.813472388)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343e7da010>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442c34c10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534426f2b10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344231c2d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344237f350>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441ebc110>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345a19d7d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534410c7f10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534405911d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343fe7bf10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534421053d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345a007350>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345d3fa350>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458b7a350>, 9884130.896003656)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458dc0b90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343fa60c10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f773510>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345863c600>, 9884133.417472292)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534429e55d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345bae0210>, 9884131.665916858)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442da3850>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345896c8a0>, 9884130.795691747)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442231b10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458745470>, 9884131.100314492)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442202010>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534403c3230>, 9884135.798012955)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441ebf750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534588603d0>, 9884136.790014423)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153443e0c3d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458863ee0>, 9884135.910170903)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440eb70d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458b7a270>, 9884131.293220501)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440589e90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458b7a3c0>, 9884132.38738943)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343fda9650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458b79940>, 9884131.865575893)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458166250>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534404cb230>, 9884130.892245283)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345d3f9a10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458c9d400>, 9884131.290388156)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343fc76010>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440591d10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441e6cde0>, 9884133.522347543)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440cdf650>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442e38dd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a4ded60>, 9884132.900719928)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442f4fa10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440ed5a90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a090360>, 9884135.320363244)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343fe9a310>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534596a36d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343ba20210>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440b614d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a4decf0>, 9884133.19794234)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343ba20850>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441c5e120>, 9884132.731256116)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440becd10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1533fadecb40>, 9884133.790453836)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343e7dafd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343ba06010>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441f8ca60>, 9884132.73729166)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343ba06650>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343e73d950>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a9331c0>, 9884135.954987034)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f43ed10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343ee01f90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a931e10>, 9884136.750312628)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343ee021d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343ee3c850>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534422c18d0>, 9884137.771154149)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343ee3c5d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f5eb510>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343ee30190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458861ef0>, 9884136.142151574)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343ba40710>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153440190210>, 9884135.438354604)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343ec1a490>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343ed06f90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15344208f000>, 9884140.551805846)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343ed05b10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343eea1990>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15344208ef20>, 9884139.098701715)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343eea1e90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343eed55d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343eeeda10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343ecf90d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343ed141d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534422c1e10>, 9884139.541511783)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343eed5c90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15344208e890>, 9884138.045937756)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343eeed8d0>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a802ac0>, 9884139.026263189)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343ecf9790>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534422c24a0>, 9884138.935607567)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343ed146d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343edc1910>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534418297f0>, 9884140.780574275)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343edc33d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343ed10c10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15344182a0b0>, 9884141.419864321)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343eea1190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343ee02390>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a8db690>, 9884142.582164291)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343ba54d50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343b8edfd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a9d8e50>, 9884141.871675072)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343b8ede50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343b894c10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a4dcde0>, 9884141.414860787)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343ed45710>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343b9563d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15344193bd20>, 9884145.386513256)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343b956790>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343b966c90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15344193b2a0>, 9884145.554185808)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343b967c10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343b592f90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15344193a820>, 9884146.563141303)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343b593e50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343b9ed750>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343b887e10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343b589190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343b8a2150>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343b5a8810>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343b5ca090>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a8dae40>, 9884143.987064408)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343b5ca750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441939010>, 9884144.052010272)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343b8a2190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534423c7150>, 9884144.02536116)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343b8d4bd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15344193a740>, 9884143.634240191)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343b886f50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441939da0>, 9884143.651805673)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343b589050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15344208c980>, 9884143.751662068)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343b5a8cd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343b661a90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343fb5c0c0>, 9884154.940351536)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343b661890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343b4c56d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343ff7f540>, 9884147.114910888)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343b4c6f50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343b4dbc90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343ff7de10>, 9884149.934230348)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343b4dbb90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343b8f3a50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343f7195c0>, 9884149.255923454)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343b8f38d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343b669fd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343b480a90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343b4a08d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15344182a580>, 9884145.661960606)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343b668350>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343fb5f690>, 9884146.105744353)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343b8e0fd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343f932430>, 9884146.894411696)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343b4a06d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343b3bfcd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343b56de10>, 9884151.749790732)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343b3bfb90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343b3d64d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343b56c2f0>, 9884150.642257668)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343b3d5fd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343b404810>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343b56f000>, 9884150.500077335)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343b404e50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343b429450>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343b56e660>, 9884151.087638084)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343b3d1990>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343b3f2350>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343b56f850>, 9884151.692428978)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343b420e50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343b440e50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343b460d90>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343b4448a0>, 9884153.229697509)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343ba440d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343b444e50>, 9884154.151467554)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343b39e810>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343b478a90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343b28b0d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343b4458d0>, 9884152.68650826)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343b473110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343b2af050>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343b671710>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343b394b50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343b39f550>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343b488190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343b59ad50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343b56d0f0>, 9884149.931334013)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343b39f410>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343b56cb40>, 9884149.392087584)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343b394a10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343f932f20>, 9884150.031643089)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343b63e950>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343b4450f0>, 9884153.744598577)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343b281ad0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15344193ad60>, 9884149.936570266)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343b5444d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343b56e0b0>, 9884150.07205465)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343b48b910>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343b29b390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343ec910d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343ec5f450>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343ec78710>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343eafac50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343ef7d490>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343eb31910>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343ecdded0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343eb46f50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343eb64150>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343ec48510>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343eba3ad0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343ebc2510>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343eb0e2d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343ecb1150>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343ebd5210>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343ea20c90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343ea49210>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343ea5c590>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343ea941d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343eab0110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343eb46c90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343eacbf10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343eacded0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343eae6910>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343e900450>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343ea1ca10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343e91a850>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343e936b50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343e966c10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343e97c0d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f291410>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343e9e8950>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343e83ead0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343ea10c90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343e879d10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343ea1e250>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343e88c150>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343e8a84d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343e8e7d50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343e721910>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343e73d090>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343e760090>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343e8d7910>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343e715e10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345896d400>, 9884131.139675852)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f37e450>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345896d550>, 9884131.183570603)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f4e44d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459b5af20>, 9884132.623431733)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f5e9590>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458c9f2a0>, 9884132.326798867)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f7c67d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343b9ef7d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343ba09b90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441749750>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343b9d0750>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343ba35fd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343ba460d0>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343ba55fd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343ba5b450>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343ba75d50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343ba40690>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343edf8d90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343ee1d5d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343ee29550>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343ee39f50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343ee5de50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343ee656d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343ee77ed0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343ee8b510>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343eea3090>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343ee30550>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343ee86550>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343eec3ed0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343eed6fd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343ed28e10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343ee66710>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343ed712d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343ed59b10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343ed80210>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343edaa1d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343ed994d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343edaef90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343edfb110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343ede3910>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343b8b03d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343b8cf6d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343b8f2890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343b915ed0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343b91b650>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343b886f10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343b8c7ad0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343b925b90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343b939b50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343b5ed5d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343b5f4490>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343b60ae90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343b5c5ad0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343b5c7f10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343b647550>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343b625350>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343b4b5590>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343b4da490>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343b50c6d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343b481450>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343b516290>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343b536090>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343b550c10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343b4943d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534422c1d30>, 9884149.935441472)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343b4944d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343f9314e0>, 9884152.571136566)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343b2aee50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441828440>, 9884149.932875255)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343b8c7990>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343b446430>, 9884152.654264078)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343b88e650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153443ed59b0>, 9884108.444114788)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343ec90ed0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153443ed5a90>, 9884109.288406922)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343ec5c4d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534421a37e0>, 9884108.595298233)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343ecabf90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534424c6510>, 9884111.884739265)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343eafab10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534424c41a0>, 9884110.51094768)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343ef7e090>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534421a3540>, 9884110.483087016)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343eb32050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534424c7a80>, 9884109.328919457)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343ecde1d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153443ed6b30>, 9884110.364225153)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343eb46e10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153443ed5080>, 9884111.763976512)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343eb64810>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534423c6ba0>, 9884110.967178673)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343ec48590>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1533faded710>, 9884110.829939771)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343eba3990>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1533fadef9a0>, 9884110.884262532)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343ebc23d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534424c51d0>, 9884109.698896091)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343eb0e250>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534424c79a0>, 9884110.706141813)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343eb8f390>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1533faded080>, 9884110.798580268)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343ebd6410>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534423c6cf0>, 9884112.527074397)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343eba2190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a801b00>, 9884112.768498912)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343ea498d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a800360>, 9884112.613184564)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343ea5c050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a8032a0>, 9884113.540207984)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343ea94890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a8026d0>, 9884112.215553736)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343eab05d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a800d00>, 9884114.578652097)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343eb47e50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1533fadec210>, 9884114.577335581)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343eacbe10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a802b30>, 9884114.620909085)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343eacfe50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a803b60>, 9884112.869604848)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343eae6fd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a6f2740>, 9884113.116134215)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343e9007d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1533fadef000>, 9884114.586979536)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343ea1c9d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a800f30>, 9884116.105604948)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343e91add0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a6f2660>, 9884115.862539256)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343e936a10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a803cb0>, 9884115.963586802)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343eb66290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a6f26d0>, 9884115.588125903)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343e97c790>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a4df460>, 9884116.23873936)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f162990>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a4ddd30>, 9884116.706425749)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343e9e8610>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a4df310>, 9884116.77832841)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343e83e650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a4dd320>, 9884116.078058792)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f033950>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a4df4d0>, 9884117.709857915)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343e878610>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a803d90>, 9884121.381727705)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343ebd6e50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a4dc7c0>, 9884117.070460124)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343e88c810>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a092eb0>, 9884117.624422632)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343e8a87d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a0912b0>, 9884117.562586626)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343e8e7b90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a092c80>, 9884118.912242768)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343e721e10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a091470>, 9884118.70873276)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343e73ce90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a091cc0>, 9884119.185742317)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343e760410>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a4dcb40>, 9884117.663588434)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343e8d7f50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a6f20b0>, 9884118.358970769)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343e715a90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459d74980>, 9884134.715528728)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343b9ef6d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a4de7b0>, 9884133.558027787)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343ba09650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1533faded390>, 9884133.406729627)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458fc8890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1533fadecbb0>, 9884133.993568188)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441b1b490>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458a71a20>, 9884134.470019542)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343ba36490>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a6f2a50>, 9884135.362513471)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343ba06c90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534421a0c20>, 9884134.167937104)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343ba55c10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a0916a0>, 9884140.737380013)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345a6a6e50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a093bd0>, 9884138.38136938)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343ba76490>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a6f2f90>, 9884134.43861799)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343ba3fb90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a0906e0>, 9884135.71053118)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343edf8b90>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a933e70>, 9884140.121404184)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343ee1da50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a930670>, 9884137.619435024)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343ee29350>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a930f30>, 9884136.301978298)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343ee380d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a4dcd00>, 9884136.841915447)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343ee5e350>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458437930>, 9884137.377541663)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343ee64290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534422c0e50>, 9884136.79128701)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343ee77dd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534423c6dd0>, 9884137.5028091)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343ee8b190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534423c43d0>, 9884138.934449455)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343eea2c50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534422c10f0>, 9884147.145288492)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343ee30310>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534422c27b0>, 9884137.35588526)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343ee86650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534422c31c0>, 9884138.377267683)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343eec3dd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534590cb380>, 9884143.850280497)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343eed71d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15344208f0e0>, 9884145.044928065)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343ed28c10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15344208d9b0>, 9884145.066327004)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343ee66050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534422c2eb0>, 9884140.687130835)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343ed07ed0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15344208fd90>, 9884140.131981501)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343eedb190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15344208d2b0>, 9884139.636496292)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343ed80710>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534422c0fa0>, 9884139.866898034)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343eda9fd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534422c1f60>, 9884139.740464702)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343eec1990>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441829710>, 9884141.382519746)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343ed83d10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441828670>, 9884142.82365923)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343ba6fc90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441829160>, 9884141.676656676)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343ede2090>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441829240>, 9884141.40169783)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343b8b09d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a6f2f20>, 9884141.930576295)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343b8cf510>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15344208d010>, 9884145.095646946)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343b8f2750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a9d95c0>, 9884143.464499027)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343b916410>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a9d87c0>, 9884146.029626478)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343b91bcd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15344182b700>, 9884141.9088137)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343b886dd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15344208fcb0>, 9884142.367515476)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343b9242d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a9d96a0>, 9884142.88366858)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343b9397d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a0932a0>, 9884144.83458202)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343b935d90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441939630>, 9884145.455869678)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343b5f6e50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343f82f150>, 9884147.095694434)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343ee1ee50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343f82ec10>, 9884146.430103851)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343b5c6190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a9d9400>, 9884145.616174592)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343ee66810>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345983fee0>, 9884147.035018092)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343b647190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343f82e7b0>, 9884148.650805123)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343b627bd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15344193a270>, 9884148.777354956)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343b4b5b90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343ff7f000>, 9884148.975260906)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343b4cee50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343f71aeb0>, 9884149.246973826)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343b50c510>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534422c3770>, 9884147.773633659)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343b481590>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534418291d0>, 9884148.246575937)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343b5164d0>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343f71b310>, 9884148.275553215)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343b5365d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15344193aa50>, 9884148.629661793)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343b550ad0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343e736610>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a801940>, 9884118.90078082)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343e737090>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343b1b6090>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345983c520>, 9884152.380578669)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343b1a8690>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343b31f710>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441938b40>, 9884153.66960498)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343b31f7d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343b33d7d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343b446c80>, 9884154.448665287)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343b1d2050>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343b311f50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343b446190>, 9884154.930087129)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343b311bd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343b463c10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153443ed7700>, 9884154.23309778)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343b462590>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343b2b9490>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343b58c610>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458d9d710>, 9884155.167716617)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343b34a1d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343ed2e210>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458d9dc50>, 9884154.885876823)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343b561250>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343b322ed0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343e70a390>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345983f850>, 9884155.147842854)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343b322450>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343eaefd50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458d9e970>, 9884155.889047926)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343eaec2d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343ea6b150>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343b405950>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153443ed7d20>, 9884156.482566375)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343ea6a610>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a2b14e0>, 9884155.403765544)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343b42ba10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343b443390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343b301a50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343b310b90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343b3123d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343b3214d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343b325850>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343b39e710>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343b18b710>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343b1b7750>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343b191d90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458d9de10>, 9884156.513009695)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343b58cb10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343b56fb60>, 9884153.593769208)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343b190bd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343f82f9a0>, 9884153.542369151)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343b2c2410>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343b445780>, 9884155.35961776)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343e709010>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343b4470e0>, 9884154.07218756)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343b1b4710>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a2b2a50>, 9884154.876756793)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343b440b50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343fd692b0>, 9884151.947544236)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343b2f7e50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343fc58590>, 9884152.15207876)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343b95ab10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534588618d0>, 9884152.937110664)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343b302410>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534402a7700>, 9884152.869788136)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343b325690>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534422c2dd0>, 9884152.95355202)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343b34dcd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441c5e580>, 9884152.0721233)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343b359c10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343ecce310>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343ec6dc90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343b56fa80>, 9884157.428791044)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343ecccbd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a2b0e50>, 9884159.7804706)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343ec6c110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343ee69c10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343b56c6e0>, 9884156.673105529)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343ee69710>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343efb7210>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a2b09f0>, 9884157.471278928)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343efb7010>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343ec16010>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a2b2660>, 9884160.29192183)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343b33ee50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343b44a850>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343efad310>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15346030b2a0>, 9884160.0025259)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343b517690>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15346030b5b0>, 9884156.730630552)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343b1c7010>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343ef62f90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458d9c6e0>, 9884157.137996448)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343ef604d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f0e2b90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343ef5b410>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345ad5c360>, 9884157.455452444)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343ef5b390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f03f210>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f1f0090>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345ad5edd0>, 9884158.207322026)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343b310a90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345ad5c8a0>, 9884157.527666535)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f1f3a90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343ef14a90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345ad5e6d0>, 9884158.113099707)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343ba2a890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f0d8750>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f044e50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345ad5f230>, 9884157.419644373)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f0db750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345ad5cec0>, 9884158.431518257)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f0470d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343ef6c1d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343ec250d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a6f1b00>, 9884157.572438257)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343e94f710>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345aabab30>, 9884160.72504678)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f1ba190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a803f50>, 9884155.752534462)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343ec24250>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f1cca10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345ad5cbb0>, 9884160.63307636)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343eb67bd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f159d10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f161f10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534580f2040>, 9884160.0077976)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f162e50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f174b90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a4df0e0>, 9884159.61631182)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f176810>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f0d7090>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458d9e430>, 9884161.245502587)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f0d6890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343edc2b10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f2b9610>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345ad5ecf0>, 9884161.26761336)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f03d490>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459a39be0>, 9884162.392359463)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f2bb150>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f11c110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f296410>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459a39400>, 9884160.372564388)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f11de90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459a3b620>, 9884160.628625343)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f295950>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f065c10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f4c1250>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459301c50>, 9884161.546288816)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f0672d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459a39fd0>, 9884159.957256846)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f4c1690>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f3f24d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459a38fa0>, 9884159.978679685)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f3f0dd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f3c4410>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459a3af20>, 9884161.472091768)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f3c7a50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f373350>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f26f410>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343b1c6590>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534580f2430>, 9884160.941909784)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f15a510>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459a3b070>, 9884160.68725544)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f373510>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459a3a430>, 9884161.741250942)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f26e0d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534580f1630>, 9884157.659562165)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f2ec1d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f0a6f50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345ad5de80>, 9884158.316592384)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f0a5650>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f1d4f90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459a3a350>, 9884160.592438472)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f1d4350>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f1ba110>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459c5f310>, 9884162.641808564)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f1bc750>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f4ad450>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f492390>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459a38830>, 9884161.9764052)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f4af410>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459c5c600>, 9884163.690549167)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f492110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343ef60d10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f4a0d10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459c5ce50>, 9884163.194254728)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f383910>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f30ad10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15346030b310>, 9884163.189159391)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f4913d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442980e10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153443ed5470>, 9884162.499493845)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f437310>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f158b10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459a3a740>, 9884165.428744191)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f158a50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f047910>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f4d3d50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345bae3d90>, 9884164.284740748)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f486390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442db0c50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345a1440d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345bae04b0>, 9884163.829701684)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442db27d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345bae2ac0>, 9884163.768236218)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345a146810>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f472610>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345bae2900>, 9884163.227808118)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f472550>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534429a5650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459d767b0>, 9884164.170015756)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f451b90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343b444360>, 9884167.751779104)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f4a1dd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534580f1e10>, 9884163.839186748)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f56ce90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459c5c910>, 9884164.33866308)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534429a4550>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344309c510>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459d75160>, 9884168.64459562)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344309c690>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f447bd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459c5cde0>, 9884164.89038545)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f447c90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534426ff2d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459d774d0>, 9884166.008173453)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f2efa10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344272cb50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459d77d90>, 9884165.295842096)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344272f150>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f56c450>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442791c50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345bae2c10>, 9884169.254737625)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344241fa10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442872d50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458a725f0>, 9884166.683599763)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442873990>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534425f4110>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345bae0520>, 9884165.91615132)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534425f4dd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442557590>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458a71940>, 9884167.231899628)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442556590>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442501890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442986bd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442e79d90>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534429a9f90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f101350>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458a72ba0>, 9884165.412357124)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442791410>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459d75a90>, 9884164.126052769)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343e6ffcd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459d77070>, 9884164.33753724)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534429a9010>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458a72ac0>, 9884165.901642172)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343efe17d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459d77620>, 9884164.407468425)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344309ccd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459d75b70>, 9884164.29559733)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343efc70d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344231fc50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458a72430>, 9884168.984837703)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534424a4290>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344248b290>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f5e83d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458314c20>, 9884168.148030072)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534421ddc90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459c5ca60>, 9884168.892521355)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f5ea690>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442cbf750>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f5c41d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534583147c0>, 9884168.694573408)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534423ac090>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458314670>, 9884167.479217824)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f5c6710>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534428e2f90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343f71a430>, 9884168.152315097)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534423edb10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441fa4510>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441d8a3d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343f719240>, 9884168.695772672)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441fa53d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458314fa0>, 9884168.07024532)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442d24d50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442180490>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343f719400>, 9884175.075561315)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442183b10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441ddd290>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441ee3b90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343f718bb0>, 9884168.5511103)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442178990>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343f719ef0>, 9884168.687589465)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441ee1510>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442203610>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458c9faf0>, 9884169.159108102)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344248a8d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534424e3090>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442286d50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f407bd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345d36c290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458314360>, 9884166.822644869)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442287b90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458315a90>, 9884165.88639094)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534424e1e90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458317e00>, 9884167.214489156)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f405f90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458a72c80>, 9884167.210483156)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f4a0510>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442554650>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441b9cb50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345ad5c210>, 9884170.272946173)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442557710>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459d75e80>, 9884172.021270722)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f1944d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458180dd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458c9d1d0>, 9884170.84769907)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458180d50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f5a3650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458c9dcc0>, 9884171.4626186)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458700510>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441719350>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345896c4b0>, 9884172.900424087)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f49bad0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441d77690>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345896fc40>, 9884171.474622713)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534428a47d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345bc68a50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345896d780>, 9884179.4830885)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345bc68710>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344248b410>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459c5faf0>, 9884172.039068324)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442489e90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458800f50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458f071d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345896f1c0>, 9884172.849743692)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442312110>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345896eac0>, 9884172.16197834)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458f06090>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345a6beed0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345896c210>, 9884171.4179816)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345a6bcc10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344224ce10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441a0b690>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441bbba90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534430f6110>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458c9ec10>, 9884169.103611317)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534430f7e10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458c9f620>, 9884169.624222476)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441bb8ad0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345896e9e0>, 9884171.077008598)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344224ee50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458c9f930>, 9884169.642340925)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f415490>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f5ebd90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345941d630>, 9884171.28838192)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441b9e690>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441701410>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345941c3d0>, 9884172.674463691)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345bc6ee50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534419e9810>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345941faf0>, 9884172.836109824)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345bcad590>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345a568d50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345941cd00>, 9884173.747899411)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345a569e50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345a3f9410>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345941c4b0>, 9884173.155680727)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345a3f8390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345bc48750>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345a4b6650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345896eba0>, 9884173.479176397)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345bc48890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345a5f6b10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534590c87c0>, 9884174.158116864)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534596c3910>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441733550>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345a339110>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343f71b850>, 9884173.860942356)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441733d90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534590ca900>, 9884173.831128577)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153459733510>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345d3cdb50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345bae3930>, 9884174.305659564)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345d3cef10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345d47fd50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345a574390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442f6a190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345f65fd90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534599c6d10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345879a390>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534590cbaf0>, 9884175.819081636)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345a4b6ad0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458317bd0>, 9884174.780385708)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345d47c410>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458a73a10>, 9884174.318556985)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345a577850>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345941e350>, 9884171.949328696)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442f69510>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345941c360>, 9884171.61312908)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345a726e90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345941d780>, 9884171.590722704)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345a62b490>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345941c980>, 9884171.779117016)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458799f50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153459829490>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458c9e580>, 9884175.50301528)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153459829dd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534583dee50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459d75be0>, 9884176.081257991)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345bc97010>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345d3da990>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534424c72a0>, 9884175.907456448)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458dc1990>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345a70e010>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534584c5750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458203bd0>, 9884175.14886385)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345a70fad0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458a722e0>, 9884175.697412524)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534584c7950>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458184b10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458200590>, 9884174.741637543)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458184790>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534596375d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458203c40>, 9884176.465643546)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153459637190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442556b50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534582009f0>, 9884176.208730588)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345a60d390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534587e63d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458201780>, 9884175.55866956)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534587e4b10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458877810>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458cc2e50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458201d30>, 9884175.652544504)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458874e50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458203e00>, 9884176.50041974)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458cc1a10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458a3b4d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534423c5ef0>, 9884177.448208965)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458a39310>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345d530b10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345a9e1490>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534423c62e0>, 9884176.616931584)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345d532f10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153459629650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1533faded8d0>, 9884174.220133424)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345a9e02d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345896cc90>, 9884175.84046304)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345d5eddd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534411a0c10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441084f90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534423c4f30>, 9884178.894340828)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534411a2550>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534423c75b0>, 9884177.10888554)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534410874d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458a97a90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440d4a1d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534590caeb0>, 9884177.166447308)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458a96dd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534423c5160>, 9884179.210544333)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440feb190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458379490>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440e32d90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534423c6510>, 9884177.364480592)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458a6ee50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441e6c980>, 9884177.280249305)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440e32710>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440e58c50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345941fc40>, 9884177.837296728)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440e58b50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534585be210>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441e6fe70>, 9884179.374982448)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534585bd750>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441025390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153459f21850>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441e6dc50>, 9884178.214772018)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441026950>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534421a02f0>, 9884179.201536432)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153459f23c10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534410002d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458203f50>, 9884178.192413716)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440fe89d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153459155d90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534429d5390>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534582000c0>, 9884178.294007095)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534590c2fd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534421a2e40>, 9884178.506023921)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442d55e90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441845390>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534421a1390>, 9884177.822104454)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345d671990>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534583c0490>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441c1d550>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534590cbb60>, 9884185.12249746)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534581a2b50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458201e80>, 9884182.280501664)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441c1c4d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534429e7290>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534423c7a80>, 9884178.95574909)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441e8bad0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440a1cb50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441f8c130>, 9884179.764553532)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440a1c290>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440b3b550>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534424c4910>, 9884180.277687157)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440b3b190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344242cad0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441f8cde0>, 9884180.89829144)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344242c410>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441b2ab10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458c9c750>, 9884179.82045276)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344234bcd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534417fc890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534423c58d0>, 9884180.173993858)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534417fe4d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344234aed0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441f8fb60>, 9884180.088149112)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442348810>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534591be4d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442335110>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441f8d710>, 9884180.995018644)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442638dd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441f8edd0>, 9884182.650409272)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442336390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534427461d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441e6ee40>, 9884182.655011307)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345d3d37d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441fe5010>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441d72580>, 9884180.473493928)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441fe6590>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441ab9b90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441b95750>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458be6350>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441d70280>, 9884181.108516358)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441aba950>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534582035b0>, 9884179.119358262)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441b94090>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458c9c600>, 9884178.934817052)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440f3ef10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441949750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441d71780>, 9884182.495100738)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345d3fb650>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440dc68d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441d70210>, 9884186.1135774)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440dc6f50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442e64950>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441d70600>, 9884182.4715906)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442e65b50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344274f050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441d71010>, 9884183.83139554)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458875d50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534408dbc90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459e77bd0>, 9884182.750295611)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534408d93d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440a14050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534421a10f0>, 9884182.700018948)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440a15550>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458951b50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442584690>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345941de10>, 9884184.032224203)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441ec9090>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442452f90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534423c70e0>, 9884183.374259396)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442452390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344207dfd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534421a17f0>, 9884182.542975262)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344207e510>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458ded9d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459e75be0>, 9884183.35343126)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441c0d850>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440ab22d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442230950>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459e777e0>, 9884183.59178369)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153443f30710>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459511e10>, 9884183.993067775)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442233210>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442340dd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441c0f290>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440af8b10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441f8d9b0>, 9884182.235661171)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442586190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441d70050>, 9884181.115534976)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534418468d0>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534595120b0>, 9884184.777701568)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442342910>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441d73cb0>, 9884181.024497192)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441c0cd10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441ceef50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459513e00>, 9884183.91436264)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344200e4d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458254910>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534581b5810>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459512ba0>, 9884184.242805447)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441193d90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459513a10>, 9884186.04077352)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458211990>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458af9bd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458ca8150>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459510050>, 9884184.847149014)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442397590>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a3b46e0>, 9884188.083116325)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458ca8a10>\n",
      "Evaluating workflow:  55%|█████▌    | 276/500 [00:50<00:30,  7.23it/s]Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458b87990>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441d73c40>, 9884185.376475953)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344251c590>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534424b5a50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345d449dd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441f8e120>, 9884185.398525402)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534424b7690>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a3b7a80>, 9884185.33876085)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345d44ac90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458c66710>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a3b44b0>, 9884185.40908302)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458c66a90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441763790>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441d72120>, 9884185.186383445)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441763a10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153443f172d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a3b4c20>, 9884185.253352333)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441cee550>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442231550>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a3b62e0>, 9884186.277863933)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442232f50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441783110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534410f0510>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442903890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442159210>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459511860>, 9884184.398978172)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442159050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a3b5e80>, 9884186.073347123)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441783fd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458860c90>, 9884186.292212209)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534410f0590>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459513620>, 9884183.462188514)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442902f10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440f07690>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534588622e0>, 9884188.421646964)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440f07710>\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.8571428571428571, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534411d8550>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440ec9190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458862660>, 9884188.064067198)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345a08d850>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440d38cd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345d43f210>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a3b58d0>, 9884187.743215656)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440d38ed0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458863a10>, 9884188.136805398)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345d43c450>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440b2a490>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153458295150>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345973a3c0>, 9884188.191363692)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440b2af10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345973b230>, 9884187.849333793)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458295d90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534409c1b10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441f8fa10>, 9884188.519057132)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345d36bc90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345d6ab3d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459510590>, 9884187.632887049)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345d6a9990>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440b86d10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440c63b50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459e74bb0>, 9884187.71936176)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440b87b50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459739470>, 9884187.910408925)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440c63c90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344267acd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459738de0>, 9884188.205429612)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442397450>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f6b9f10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534408dbad0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441005790>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a3b6b30>, 9884187.587337097)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440ec8d10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458862120>, 9884186.094206356)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441006010>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345973a7b0>, 9884190.799364014)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f6bb190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441d720b0>, 9884185.896247879)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345d547010>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440f7ccd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344180da90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345973b380>, 9884188.794275736)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f61b050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441d70d00>, 9884190.920696925)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344180f390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f6383d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a1987c0>, 9884189.839272244)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441902a90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f65bc10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345a598c50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a19b540>, 9884195.512661876)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f65b510>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534421a2190>, 9884189.037571585)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345a598710>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f72b450>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a19a200>, 9884189.67527926)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f72bd50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440eb63d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459512580>, 9884190.227532608)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153459679ed0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441cef490>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f8c1390>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a19aeb0>, 9884190.127163297)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441781410>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a198d70>, 9884190.595580809)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f8c2d90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f78b390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f874e50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a1996a0>, 9884190.887984212)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f7881d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f837590>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440f07f10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f71ec50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459739a20>, 9884190.966666844)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440f04290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a4cb4d0>, 9884189.876656236)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f71ead0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f77df50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f9c4b90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f603510>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441d73b60>, 9884190.596842883)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f8740d0>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458862f90>, 9884190.787759759)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153458e262d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a4cb230>, 9884192.95666191)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f9e6e50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a4c8ec0>, 9884191.802033488)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f9c4510>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a19a190>, 9884189.13697102)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534581c8f50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345d352090>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f939cd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a1986e0>, 9884193.457177345)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f9ae650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a4c8440>, 9884192.785521531)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f939d90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f920090>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a4c8ad0>, 9884191.747086188)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f921390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f730050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345993c4b0>, 9884191.467246143)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f733510>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f728750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345993db70>, 9884194.404574247)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343fad9c10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343fa8e350>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345993f9a0>, 9884192.29846428)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343fa8fc90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343fa6d4d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345993c440>, 9884194.683591831)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343fa6f890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f7e4450>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458862e40>, 9884192.421573287)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153459166e50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343fab1090>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459e771c0>, 9884193.703946676)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343fa04b90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343fb97750>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440f045d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345993f2a0>, 9884192.812222453)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343fb97890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345993ea50>, 9884194.277606124)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440f07d10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343fb7dbd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343fb62010>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345962f000>, 9884192.831496608)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343fb7d850>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345962db00>, 9884192.840988718)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343fb630d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343fccadd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f97fad0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a4c80c0>, 9884191.478861948)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f959c10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a4cbb60>, 9884193.83201502)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343fcc8e10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343fc7c250>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345962ec10>, 9884194.511978095)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343fc7ce50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343fbb76d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343fa2be90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345962eeb0>, 9884194.423799315)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343fb52c50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343fc1ca90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345962e200>, 9884198.393312372)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343fc1d290>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343fc31290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458435860>, 9884198.368628751)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343fc30a10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343fda9b90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345962e970>, 9884194.793440668)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343fdaa2d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343fd63ed0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343fd07550>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458435780>, 9884196.339914396)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343fd627d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534584360b0>, 9884195.1186696)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343fd079d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343fdab110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345d5c9210>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a4c8fa0>, 9884195.865246218)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343fda97d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458435d30>, 9884194.66568272)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15345d5c91d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343fdc9cd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a4ca4a0>, 9884196.417714912)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343fdca590>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343fe7a0d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458436740>, 9884197.096769275)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343fdb1590>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343fd92a50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345962df60>, 9884195.567962691)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343fe302d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343fbf28d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343fc9ee90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a4cb700>, 9884196.194911826)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343fa2b150>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345993f8c0>, 9884197.024247032)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343fd93bd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345962fd90>, 9884193.111401675)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343fbf27d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a3b65f0>, 9884193.2856675)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343fc9d510>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343fd2cc10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343ff6f790>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343ff28490>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459f8db00>, 9884199.61363094)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343ff6f510>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459f8e190>, 9884197.162924351)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343ff2b190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343ff52190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459f8f380>, 9884197.82501412)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343ff51d10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343fa6dd10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345962e120>, 9884197.776559982)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343ff6c550>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440065890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534585351d0>, 9884199.253463613)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440066450>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344004d490>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458535390>, 9884199.930304565)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534400bced0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534400bd090>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345993d5c0>, 9884200.73156644)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534400bc110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534400d9010>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534401cacd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459f8cfa0>, 9884200.405424643)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343fdab290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345973acf0>, 9884197.271220328)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343fd2e790>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343ffc69d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343ff85a90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343fe16c10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534400d35d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a3b7230>, 9884196.363399124)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343ff86b50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458537380>, 9884199.3688625)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534401c9710>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459f8e040>, 9884195.98890362)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343ffc4fd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345993c9f0>, 9884196.648797808)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343fe15750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a4cbee0>, 9884196.639212124)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534400d1f10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343fc1c9d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534584355c0>, 9884198.612912076)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440186e50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343fe82ad0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459f8c7c0>, 9884199.182129094)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343fb7ee50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343ffc42d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441b57d90>, 9884199.67333481)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344019ea10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534402a1890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343fa20e90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534585347c0>, 9884199.559333041)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534401d3190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441b57a10>, 9884200.013941621)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534402bc710>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343ff51e10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441b56120>, 9884199.950054027)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344019f750>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534400f8190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440241150>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345962c210>, 9884201.932369389)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534400f8510>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441b56040>, 9884200.012562357)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440242650>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534401fbe90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a4c9e80>, 9884201.275780348)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534401f9c90>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534400ab490>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534584351d0>, 9884201.99310464)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534400a8090>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440273450>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534585358d0>, 9884201.179072984)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344039c410>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343fd42b10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440183bd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344002c150>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534402b1fd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344001a550>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345962e270>, 9884205.058127834)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534402b0550>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458536660>, 9884199.08499971)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344001a4d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534590d8b40>, 9884201.38791796)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534403c7690>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458535b70>, 9884199.370060192)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440181090>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a4c8520>, 9884198.749252299)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344000c290>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440311a50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534590da270>, 9884210.008041762)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440310210>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344012e2d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345993cde0>, 9884201.79520698)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534403e6e50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440362f50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440351250>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534590db4d0>, 9884203.14157012)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440362e10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534590d80c0>, 9884203.053880228)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534403514d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440488bd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534584350f0>, 9884203.538230153)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344047b850>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343ff51b10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534590db310>, 9884203.887876678)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343ff524d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440473c50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440441650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534590d9b00>, 9884201.794018684)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440472710>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345962d1d0>, 9884202.353548588)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440442390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534400bc210>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441c5e0b0>, 9884203.756270355)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344072b190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343feb8350>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440406450>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441c5c590>, 9884202.751719106)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343ffd1b10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441c5c4b0>, 9884202.219282685)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440405790>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153441067490>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441c5fcb0>, 9884203.136217969)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534410644d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534404daf90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459f8e970>, 9884205.671172244)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534404d91d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440c9e5d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440265e50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534590d9da0>, 9884201.698274948)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534402d6e90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a199ef0>, 9884203.726535343)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440c9e490>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534404bde50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344092c690>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441c5ef20>, 9884205.061570648)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440357710>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441c5fc40>, 9884205.981053764)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344092c310>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344093a450>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344054bd50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441b547c0>, 9884205.32174011)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344093a550>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458eb1d30>, 9884204.6420492)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344054ab90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534405390d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459f8e3c0>, 9884206.592611892)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440539450>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440bd1c50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440525a90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458eb06e0>, 9884203.94369049)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440bd0890>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344039a310>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458eb1240>, 9884206.751227887)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344054cc90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344047b8d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534406d8750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458eb1e10>, 9884207.172808444)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344047b890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a5f16a0>, 9884205.022797609)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534406d9a10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534406c0150>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a5f0910>, 9884205.798963146)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534406c0c10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440aa6790>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a5f2510>, 9884206.305261554)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440aa4f10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344069e550>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440ae7410>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534404bcf50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534590d8360>, 9884205.582207104)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440524d50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534585366d0>, 9884206.245674357)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15344069e410>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441c5d940>, 9884204.116128968)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440ae4e90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458eb3620>, 9884203.706186865)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534404bcd50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534400f8390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440866ad0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a5f35b0>, 9884206.799490988)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534407df8d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a5f2b30>, 9884207.632251209)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440603190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534408a4510>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534407b2810>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a5f2200>, 9884207.447647916)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534408a4610>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a5f0a60>, 9884208.814175447)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534407b0510>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534405cd8d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a5f2a50>, 9884207.178711792)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440440b10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343b36d390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343a79c890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345973a190>, 9884207.453551436)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343b36c2d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343b1cbc90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458fc2270>, 9884213.406153157)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343b1cb150>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343a7bad10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458fc22e0>, 9884207.814586507)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343a7ba850>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343a7c0c50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458fc0c90>, 9884208.749187095)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343a7c1050>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343a7d2750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458eb1c50>, 9884209.513838142)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343a7d2650>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343a7c3090>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534590dbee0>, 9884207.651039122)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343a7c3510>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440bd9f90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343a828850>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a5f2dd0>, 9884208.741888111)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343a7b8c50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458fc3380>, 9884210.166373849)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343a828f10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343a7f9510>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440806dd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458fc0980>, 9884209.38970886)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343a79c950>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a5f3d90>, 9884206.666781705)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440805950>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a3e3770>, 9884219.980400017)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343a7f9250>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440f2acd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a3e2350>, 9884208.802575871)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440f2a610>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343a801610>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343a879810>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a5f3000>, 9884211.118004085)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343a801510>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a3e3e70>, 9884213.220533025)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343a879610>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343a6ad5d0>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a3e32a0>, 9884210.35696078)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343a6adf10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f7599d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343a6ccc90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a3e00c0>, 9884210.307077423)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343a6b8690>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a3e3b60>, 9884212.650871629)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343a6ccb50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343a6e3a90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441c5e040>, 9884211.003418831)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343a6e3590>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343a6ff550>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441a44750>, 9884211.129282026)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343a6ff410>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343a79df90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343a7416d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441a44830>, 9884211.179106481)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343a79e0d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441a475b0>, 9884211.18026392)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343a741d90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343a749250>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441a47380>, 9884210.905700965)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343a749110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343a74b390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343a680850>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343a852cd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343a6aaed0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a3e0670>, 9884209.394454008)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343a6ab5d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441a457f0>, 9884212.45142104)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343a74b690>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a3e3230>, 9884209.700811764)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343a680650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441a46660>, 9884210.21671282)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343a852890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440467650>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440373ad0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345bc56490>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534596de310>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15345d5e50d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153442203e50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f43fd90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f2d4ed0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f053090>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a2b1cc0>, 9884222.39914782)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343b35a850>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343a72c7d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459f8da20>, 9884222.286995476)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440464a50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345993cad0>, 9884222.214615667)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440373410>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459f8ed60>, 9884223.35261208)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534597aa3d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459f8c4b0>, 9884222.31975497)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534590916d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15346030a5f0>, 9884222.65450487)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441a17550>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345993e3c0>, 9884222.86042378)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534425f5490>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a2b3d20>, 9884222.621176967)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f43ed50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a2b0280>, 9884229.685898582)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f177810>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459e74ad0>, 9884222.282658096)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f22fb50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441a478c0>, 9884211.768501792)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343a764850>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343a591450>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441a45630>, 9884214.930092411)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343a58b190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343b527a90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343b61f2d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343ee2a4d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1534425d0c50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f380cd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345896cc20>, 9884222.575487817)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534428c5290>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343ebc2810>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440b10090>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f767ad0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15344090a250>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343fa17310>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534590daba0>, 9884222.54270639)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343f7663d0>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534590d9e10>, 9884223.356959224)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1534409099d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a2b38c0>, 9884223.21453445)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343b527f50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345896f000>, 9884222.314465668)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343b61f450>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345896ec80>, 9884222.473145)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343ed96810>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345896d320>, 9884222.68205276)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153441e40210>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534590d8a60>, 9884223.297681507)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440c63190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534590d8c20>, 9884228.170711135)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153442948310>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458317000>, 9884224.473187491)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343fa164d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343a5fe090>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f0076d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458201470>, 9884225.9338525)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343b4940d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458fc0c20>, 9884224.378986469)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343ba5a810>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153443127010>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343a60d310>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458fc1ef0>, 9884230.537084924)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153443011c10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458fc0210>, 9884224.471515317)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343a60b190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343a83ec50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343a627810>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a5f0c20>, 9884224.430790836)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343a83e450>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a5f1780>, 9884225.180581203)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343a6276d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343f1bfd90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343a6323d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345962ef20>, 9884225.283124996)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343ef3b2d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a2b2890>, 9884229.49893108)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343a6321d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343a640dd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441a477e0>, 9884225.553919785)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343a63b190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343a643450>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343a64bc10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441a473f0>, 9884224.714765033)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343a643310>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441a443d0>, 9884225.2995642)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343a64bad0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343a6538d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458eb27b0>, 9884226.850674346)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343a6536d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343e8e5910>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343a663cd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441a454e0>, 9884225.667952484)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153459ab5810>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441a45470>, 9884226.882474996)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343a663b90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343a66dd90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343a676410>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a3e3ee0>, 9884228.151251577)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343a66e0d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a3e10f0>, 9884228.340399705)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343a6762d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343b796290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a3e37e0>, 9884227.698300513)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343b796850>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343b781f10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343b7a1050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458d9c130>, 9884227.01424849)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343a67b290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458fc0750>, 9884231.55957273)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343a7e9090>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343b791ed0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153443f1ab30>, 9884228.263496356)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343b7b2190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343a613190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153443f1b2a0>, 9884226.425750736)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343b790750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a5f3930>, 9884224.273418348)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343b9d8b90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343b7c7990>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a5f1e80>, 9884229.147970393)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343b7c7850>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343b7dde90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458fc0130>, 9884238.504008967)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343edd1c10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343b7b1950>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343b834810>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1533fae19860>, 9884229.662907606)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343a645310>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458d9d550>, 9884230.43255484)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343b82b190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343b802650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459b58440>, 9884231.446577944)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343b802c90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343b8252d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153441a47d20>, 9884229.72413916)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343b825210>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343b7fb310>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345863d780>, 9884230.097584432)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343b7fb190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343b875a10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343b7b6950>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343b7ea850>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343b7fa5d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343a71d7d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343a80a910>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153443ed7e00>, 9884228.54842725)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343b7dd2d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459b59a20>, 9884230.22456456)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343b875690>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1533fae19010>, 9884228.384110563)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343b7b6710>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1533fae18980>, 9884228.72167159)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343b7ea710>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1533fae19a20>, 9884228.03883225)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343b7f8450>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153458fc2740>, 9884229.303832369)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343b7c65d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343b682c10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153440b10b50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343b694f50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345863d1d0>, 9884231.067635853)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153440f3dc10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1533fae18de0>, 9884231.965461913)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343b694d50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343a658610>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345863ce50>, 9884232.73055436)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343b6abfd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343b6c3750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153443ed7690>, 9884234.010092476)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343b7ec410>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343a653910>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343b6e7010>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459b5be70>, 9884231.37190003)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343b6b2710>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343f82edd0>, 9884232.89469908)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343b6e6ed0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343b81b110>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345962d0f0>, 9884232.999912575)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343b6dee50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343b6cb390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343b706a90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1534583151d0>, 9884231.723734848)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343b6cbe50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343f82d400>, 9884232.662895272)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343b706950>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343b715850>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343f82c280>, 9884232.856248904)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343b715690>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343b859110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343b732cd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343f82e890>, 9884233.397441652)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343b859150>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343f82c9f0>, 9884233.705642948)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343b732b90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343b73c7d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345863d240>, 9884232.689119704)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343b73cc90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343b750110>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345863edd0>, 9884233.356279766)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343ebfac90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343b6b74d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343f931d30>, 9884232.478877136)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343b750610>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1533fae182f0>, 9884231.885941263)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343b6b7110>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343b857d50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459b58050>, 9884230.16375029)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343b7d8e10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343b751c50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343f932c10>, 9884234.80789989)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343b751ed0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343b76dcd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343f932740>, 9884235.305663165)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343b76e1d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343a488650>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343a495790>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343f931be0>, 9884233.384422172)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343a488950>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a5f0360>, 9884238.86582273)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343a495650>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343a4b01d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153443f1a580>, 9884234.504717221)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343a4b06d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343a4ac210>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343a4c89d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343f82c6e0>, 9884234.239989229)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343a4ac550>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153459d76510>, 9884239.9839336)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343a4c8890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343a4e3b10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343fa39d30>, 9884237.042820416)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343a4e3fd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343a4e9b90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343fa39160>, 9884235.5086148)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343b859ad0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343a4f2810>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343a50dfd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343fa39630>, 9884237.740267348)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343b732950>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343fa3a190>, 9884234.98755772)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343a50e710>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343a51b610>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343a528d10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343a482610>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343fa3b000>, 9884236.25325478)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343a519090>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343fa39320>, 9884235.645996844)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343a529c90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343f932ac0>, 9884233.739158276)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343a482410>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343a4ca510>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343a544650>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343a4f0dd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15343a5565d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15345a801400>, 9884235.442712516)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343b733190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343fa3a740>, 9884235.136751294)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343a544d50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343fa38830>, 9884236.509957382)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343a4f0e10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15343fa380c0>, 9884236.527729373)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15343a5563d0>\n",
      "Evaluating workflow:  56%|█████▌    | 278/500 [00:51<00:34,  6.42it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  56%|█████▌    | 279/500 [00:51<00:33,  6.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  56%|█████▌    | 281/500 [00:51<00:30,  7.29it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  56%|█████▋    | 282/500 [00:51<00:34,  6.29it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.6, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  57%|█████▋    | 283/500 [00:52<01:11,  3.04it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  57%|█████▋    | 284/500 [00:52<01:05,  3.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.23529411764705882, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  57%|█████▋    | 286/500 [00:53<00:57,  3.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  58%|█████▊    | 288/500 [00:53<00:40,  5.29it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.2857142857142857, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.33333333333333337, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  58%|█████▊    | 292/500 [00:54<00:33,  6.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  59%|█████▊    | 293/500 [00:54<00:38,  5.42it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  59%|█████▉    | 296/500 [00:55<00:37,  5.38it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.5454545454545454, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.8, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  60%|█████▉    | 299/500 [00:55<00:30,  6.57it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  60%|██████    | 301/500 [00:55<00:25,  7.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  61%|██████    | 304/500 [00:56<00:27,  7.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.3333333333333333, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  61%|██████    | 305/500 [00:56<00:32,  6.05it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  61%|██████▏   | 307/500 [00:57<00:44,  4.36it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.26666666666666666, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.5, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  62%|██████▏   | 309/500 [00:57<00:33,  5.77it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.11764705882352941, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  62%|██████▏   | 310/500 [00:57<00:40,  4.67it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.4, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  62%|██████▏   | 311/500 [00:57<00:43,  4.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  62%|██████▏   | 312/500 [00:58<00:56,  3.35it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  63%|██████▎   | 316/500 [00:59<00:39,  4.61it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  63%|██████▎   | 317/500 [00:59<00:47,  3.84it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.8, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  64%|██████▍   | 322/500 [00:59<00:23,  7.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.5714285714285715, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  65%|██████▍   | 324/500 [01:00<00:21,  8.16it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  65%|██████▌   | 327/500 [01:01<00:37,  4.61it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.4, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.8, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  66%|██████▌   | 330/500 [01:01<00:27,  6.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.5714285714285715, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  66%|██████▌   | 331/500 [01:01<00:30,  5.46it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  67%|██████▋   | 335/500 [01:02<00:23,  7.07it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  68%|██████▊   | 339/500 [01:02<00:17,  9.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.3157894736842105, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.8, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.2857142857142857, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  68%|██████▊   | 341/500 [01:03<00:25,  6.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.5454545454545454, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  69%|██████▉   | 345/500 [01:03<00:22,  6.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.4, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  69%|██████▉   | 346/500 [01:04<00:31,  4.96it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  69%|██████▉   | 347/500 [01:04<00:33,  4.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  70%|██████▉   | 349/500 [01:05<00:41,  3.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.3333333333333333, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.8, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  71%|███████   | 353/500 [01:05<00:21,  6.74it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.33333333333333337, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  71%|███████   | 355/500 [01:05<00:24,  6.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.23529411764705882, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.5, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  72%|███████▏  | 360/500 [01:06<00:15,  9.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.3333333333333333, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.8, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.6363636363636364, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  72%|███████▏  | 362/500 [01:06<00:13, 10.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.5714285714285715, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  73%|███████▎  | 364/500 [01:06<00:14,  9.60it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  73%|███████▎  | 366/500 [01:07<00:34,  3.87it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  74%|███████▍  | 369/500 [01:08<00:26,  4.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.8571428571428571, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.8, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.75, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  74%|███████▍  | 372/500 [01:08<00:19,  6.60it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  75%|███████▍  | 373/500 [01:08<00:24,  5.09it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  75%|███████▌  | 377/500 [01:09<00:18,  6.52it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.06060606060606061, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  76%|███████▌  | 380/500 [01:09<00:16,  7.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.4444444444444445, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  77%|███████▋  | 384/500 [01:10<00:15,  7.57it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.11764705882352941, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  77%|███████▋  | 385/500 [01:10<00:15,  7.58it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  77%|███████▋  | 386/500 [01:10<00:18,  6.16it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  78%|███████▊  | 388/500 [01:11<00:27,  4.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.16, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.5, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  78%|███████▊  | 391/500 [01:11<00:16,  6.77it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.8, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  79%|███████▊  | 393/500 [01:12<00:17,  6.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  79%|███████▉  | 396/500 [01:12<00:16,  6.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.2857142857142857, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  80%|███████▉  | 399/500 [01:13<00:18,  5.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.5714285714285715, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  80%|████████  | 402/500 [01:13<00:12,  7.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  81%|████████  | 404/500 [01:13<00:10,  9.06it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.4444444444444445, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  81%|████████  | 406/500 [01:14<00:11,  8.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.33333333333333337, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  82%|████████▏ | 409/500 [01:14<00:14,  6.37it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.3333333333333333, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.25, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  82%|████████▏ | 411/500 [01:15<00:14,  6.16it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.5, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.18181818181818182, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  83%|████████▎ | 414/500 [01:15<00:12,  6.89it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.8, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  83%|████████▎ | 416/500 [01:15<00:11,  7.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.5, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.8, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.4, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  84%|████████▍ | 419/500 [01:16<00:13,  5.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  84%|████████▍ | 420/500 [01:16<00:18,  4.42it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  85%|████████▍ | 423/500 [01:17<00:14,  5.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  85%|████████▌ | 425/500 [01:17<00:10,  7.05it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.19999999999999998, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  85%|████████▌ | 427/500 [01:17<00:14,  5.14it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.6, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.8571428571428571, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  86%|████████▌ | 429/500 [01:18<00:10,  6.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.5, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.2857142857142857, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  86%|████████▌ | 430/500 [01:18<00:12,  5.66it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  86%|████████▌ | 431/500 [01:18<00:12,  5.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.4444444444444445, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  87%|████████▋ | 435/500 [01:18<00:07,  9.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.5, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  88%|████████▊ | 439/500 [01:19<00:07,  7.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  88%|████████▊ | 441/500 [01:20<00:13,  4.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  89%|████████▊ | 443/500 [01:20<00:12,  4.66it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  89%|████████▉ | 445/500 [01:21<00:10,  5.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  89%|████████▉ | 446/500 [01:21<00:10,  5.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.8235294117647058, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.5454545454545454, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  90%|█████████ | 450/500 [01:21<00:07,  6.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.5, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.1111111111111111, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.5263157894736842, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.4444444444444445, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  90%|█████████ | 452/500 [01:21<00:05,  8.48it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.2857142857142857, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  91%|█████████ | 456/500 [01:22<00:05,  8.04it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  92%|█████████▏| 459/500 [01:23<00:08,  4.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  92%|█████████▏| 460/500 [01:23<00:08,  4.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  92%|█████████▏| 461/500 [01:24<00:08,  4.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.6153846153846153, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  93%|█████████▎| 467/500 [01:24<00:03,  9.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  94%|█████████▍| 469/500 [01:24<00:04,  7.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  94%|█████████▍| 471/500 [01:25<00:04,  6.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  95%|█████████▌| 475/500 [01:25<00:02,  9.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.47619047619047616, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  95%|█████████▌| 477/500 [01:26<00:03,  7.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.058823529411764705, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  96%|█████████▌| 479/500 [01:26<00:03,  6.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.4444444444444445, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  96%|█████████▌| 480/500 [01:26<00:02,  6.72it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.5714285714285715, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  96%|█████████▋| 482/500 [01:26<00:03,  5.77it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.2857142857142857, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  97%|█████████▋| 484/500 [01:27<00:03,  5.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.8, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.375, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  97%|█████████▋| 486/500 [01:27<00:02,  5.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  98%|█████████▊| 488/500 [01:28<00:02,  5.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.14285714285714288, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0.33333333333333337, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  98%|█████████▊| 491/500 [01:28<00:01,  7.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.2222222222222222, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  98%|█████████▊| 492/500 [01:28<00:01,  5.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  99%|█████████▊| 493/500 [01:29<00:01,  4.71it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  99%|█████████▉| 494/500 [01:29<00:01,  3.84it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.5714285714285715, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0.8, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  99%|█████████▉| 496/500 [01:30<00:01,  3.77it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.33333333333333337, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  99%|█████████▉| 497/500 [01:31<00:01,  2.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.0625, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow: 100%|█████████▉| 498/500 [01:31<00:00,  2.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow: 100%|█████████▉| 499/500 [01:33<00:00,  1.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0.4, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow: 100%|██████████| 500/500 [01:38<00:00,  5.05it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "for idx in range(0,5):\n",
    "    textgrad_optimizer = TextGradOptimizer(\n",
    "        graph=workflow_graph, \n",
    "        optimize_mode=\"all\",\n",
    "        executor_llm=executor_llm, \n",
    "        optimizer_llm=optimizer_llm,\n",
    "        batch_size=3,\n",
    "        max_steps=20,\n",
    "        evaluator=evaluator,\n",
    "        eval_every_n_steps=1,\n",
    "        eval_rounds=1,\n",
    "        save_interval=None,\n",
    "        save_path=\"./\",\n",
    "        rollback=True,\n",
    "        constraints=[]\n",
    "    )\n",
    "    with suppress_logger_info():\n",
    "        results = textgrad_optimizer.evaluate(dataset=benchmark, eval_mode=\"test\")\n",
    "        out_dict = textgrad_optimizer.evaluator._evaluation_records\n",
    "        renewed_dict = {}\n",
    "        for key in out_dict.keys():\n",
    "            renewed_dict[key] = [out_dict[key]['prediction'], out_dict[key]['label'], out_dict[key]['metrics']]\n",
    "#     len(textgrad_optimizer.evaluator._evaluation_records)\n",
    "    with open(f\"./hotpot_5times_4omini_{idx}.pkl\", 'wb') as file:\n",
    "        pickle.dump(renewed_dict, file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "f53f86c3",
   "metadata": {},
   "outputs": [],
   "source": [
    "# textgrad_optimizer.evaluator._evaluation_records[list(textgrad_optimizer.evaluator._evaluation_records.keys())[0]]\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "1b29b5fc",
   "metadata": {},
   "outputs": [],
   "source": [
    "d_full = []\n",
    "for idx in range(0,5):\n",
    "    d_full.append(pd.read_pickle(f\"./hotpot_5times_4omini_{idx}.pkl\"))\n",
    "out_score = []\n",
    "for prob in d_full[0].keys():\n",
    "    for iteritem in d_full:\n",
    "        if iteritem[prob][2]['acc'] == 1.0:\n",
    "            out_score.append(iteritem[prob][2])\n",
    "            break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "44036f12",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.5303038027332723"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.DataFrame(out_score)['f1'].sum() / 500"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "f8e7c3c0",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.438"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.DataFrame(out_score)['em'].sum() / 500"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "14276527",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.708"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.DataFrame(out_score)['acc'].sum() / 500"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "b59c472f",
   "metadata": {},
   "outputs": [],
   "source": [
    "#generated_workflow \n",
    "qa_workflow = {\n",
    "    \"goal\": \"Provide a concise answer to the question using relevant context. The answer must be straightforward and avoid unnecessary explanations.\",\n",
    "    \"tasks\": [\n",
    "        {\n",
    "            \"name\": \"generate_answer\",\n",
    "            \"description\": \"Extract and formulate an answer from the given context.\",\n",
    "            \"inputs\": [\n",
    "                {\"name\": \"question\", \"type\": \"str\", \"required\": True, \"description\": \"The question that needs to be answered.\"},\n",
    "                {\"name\": \"context\", \"type\": \"str\", \"required\": True, \"description\": \"The background information pertinent to the question.\"}\n",
    "            ],\n",
    "            \"outputs\": [\n",
    "                {\"name\": \"answer\", \"type\": \"str\", \"required\": True, \"description\": \"The direct answer to the question.\"}\n",
    "            ],\n",
    "            \"prompt_template\": StringTemplate(instruction=\"Use the context to determine the best answer to the question. Provide your final answer in a clear format, without extra commentary or reasoning.\"),\n",
    "            \"parse_mode\": \"xml\"\n",
    "        }\n",
    "    ]\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "2c62cea3",
   "metadata": {},
   "outputs": [],
   "source": [
    "api_key = \"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\"\n",
    "# Function to encode the image\n",
    "import os\n",
    "os.environ[\"OPENAI_API_KEY\"] = api_key\n",
    "OPENAI_API_KEY = os.getenv(\"OPENAI_API_KEY\")\n",
    "\n",
    "\n",
    "llm_config = OpenAILLMConfig(model=\"gpt-4o-mini-2024-07-18\", openai_key=OPENAI_API_KEY, top_p=0.85, temperature=0.2, frequency_penalty=0.0, presence_penalty=0.0)\n",
    "llm = OpenAILLM(config=llm_config)\n",
    "executor_llm = OpenAILLM(config=llm_config)\n",
    "optimizer_llm = OpenAILLM(config=llm_config)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "d965a023",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2026-01-04 10:44:39.340\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.benchmark.pertqa\u001b[0m:\u001b[36m_load_data_from_file\u001b[0m:\u001b[36m52\u001b[0m - \u001b[1mloading HotPotQA data from /home/tl688/pitl688/selfevolve/EvoAgentX/examples/pertqa/reploge_train.json ...\u001b[0m\n",
      "\u001b[32m2026-01-04 10:44:39.427\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.benchmark.pertqa\u001b[0m:\u001b[36m_load_data_from_file\u001b[0m:\u001b[36m52\u001b[0m - \u001b[1mloading HotPotQA data from /home/tl688/pitl688/selfevolve/EvoAgentX/examples/pertqa/reploge_train.json ...\u001b[0m\n",
      "\u001b[32m2026-01-04 10:44:39.503\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.benchmark.pertqa\u001b[0m:\u001b[36m_load_data_from_file\u001b[0m:\u001b[36m52\u001b[0m - \u001b[1mloading HotPotQA data from /home/tl688/pitl688/selfevolve/EvoAgentX/examples/pertqa/reploge_test.json ...\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "from evoagentx.benchmark import PertQA\n",
    "from copy import deepcopy\n",
    "\n",
    "import nest_asyncio\n",
    "nest_asyncio.apply()\n",
    "\n",
    "def collate_func(example: dict) -> dict:\n",
    "    problem = \"Question: {}\\n\\nAnswer:\".format(example[\"question\"])\n",
    "    return {\"problem\": problem}\n",
    "\n",
    "\n",
    "hotpotqa_graph_data = {\n",
    "    \"goal\": \"Answer the question based on the context. The answer should be a direct response to the question, without including explanations or reasoning.\",\n",
    "    \"tasks\": [\n",
    "        {\n",
    "            \"name\": \"answer_generate\",\n",
    "            \"description\": \"Answer the question based on the context.\",\n",
    "            \"inputs\": [\n",
    "                {\"name\": \"problem\", \"type\": \"str\", \"required\": True, \"description\": \"The problem to solve.\"}\n",
    "            ],\n",
    "            \"outputs\": [\n",
    "                {\"name\": \"answer\", \"type\": \"str\", \"required\": True, \"description\": \"The answer to the problem.\"}\n",
    "            ],\n",
    "            \"prompt_template\": StringTemplate(instruction=\"Think step by step to answer the question. You should explain your thinking process in the 'thought' field, and provide the final answer in the 'answer' field.\\nFormat your output in xml format, such as <thought>xxx</thought> and <answer>xxx</answer>.\"),\n",
    "            \"parse_mode\": \"xml\"\n",
    "        }\n",
    "    ] \n",
    "}\n",
    "\n",
    "os.environ[\"AZURE_OPENAI_DEPLOYMENT_NAME\"] = \"gpt-4o-mini\"\n",
    "os.environ[\"AZURE_OPENAI_ENDPOINT\"] = \"https://tianyuliu-hua-raredisea-resource.cognitiveservices.azure.com/\"\n",
    "os.environ[\"AZURE_OPENAI_KEY\"] = \"2pa9h2ZIN1lQepFWwYADlXIKIansa9KPhxMoumeGbRQ08f2uDTXiJQQJ99BKACHYHv6XJ3w3AAAAACOGsQIt\"\n",
    "os.environ[\"AZURE_OPENAI_API_VERSION\"] = \"2025-01-01-preview\"\n",
    "llm_config = LiteLLMConfig(model=\"azure/\" + os.getenv(\"AZURE_OPENAI_DEPLOYMENT_NAME\"),  # Azure model format\n",
    "    azure_endpoint=os.getenv(\"AZURE_OPENAI_ENDPOINT\"),\n",
    "    azure_key=os.getenv(\"AZURE_OPENAI_KEY\"),\n",
    "    api_version=os.getenv(\"AZURE_OPENAI_API_VERSION\", \"2024-12-01-preview\"), top_p=0.85, temperature=0.2, frequency_penalty=0.0, presence_penalty=0.0)\n",
    "\n",
    "executor_llm = LiteLLM(config=llm_config)\n",
    "optimizer_llm = LiteLLM(config=llm_config)\n",
    "llm = executor_llm\n",
    "\n",
    "benchmark = PertQA(pertdata='reploge')\n",
    "workflow_graph = SequentialWorkFlowGraph.from_dict(hotpotqa_graph_data)\n",
    "agent_manager = AgentManager()\n",
    "agent_manager.add_agents_from_workflow(workflow_graph, executor_llm.config)\n",
    "\n",
    "evaluator = Evaluator(\n",
    "    llm=executor_llm, \n",
    "    agent_manager=agent_manager, \n",
    "    collate_func=collate_func, \n",
    "    num_workers=20, \n",
    "    verbose=True\n",
    ")\n",
    "\n",
    "textgrad_optimizer = TextGradOptimizer(\n",
    "    graph=workflow_graph, \n",
    "    optimize_mode=\"all\",\n",
    "    executor_llm=executor_llm, \n",
    "    optimizer_llm=optimizer_llm,\n",
    "    batch_size=3,\n",
    "    max_steps=20,\n",
    "    evaluator=evaluator,\n",
    "    eval_every_n_steps=1,\n",
    "    eval_rounds=1,\n",
    "    save_interval=None,\n",
    "    save_path=\"./\",\n",
    "    rollback=True,\n",
    "    constraints=[]\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "67291165",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3000"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(benchmark._test_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "76b45e2f",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2026-01-04 10:44:51.125\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36m__main__\u001b[0m:\u001b[36m<module>\u001b[0m:\u001b[36m1\u001b[0m - \u001b[1mEvaluating workflow on test set...\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   0%|          | 1/3000 [00:01<1:14:08,  1.48s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   0%|          | 4/3000 [00:01<17:10,  2.91it/s]  "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   0%|          | 13/3000 [00:02<03:55, 12.66it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   1%|          | 18/3000 [00:02<03:01, 16.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   1%|          | 21/3000 [00:03<07:44,  6.42it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   1%|          | 28/3000 [00:03<04:41, 10.55it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   1%|          | 32/3000 [00:04<03:38, 13.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   1%|          | 35/3000 [00:04<03:30, 14.07it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   1%|▏         | 39/3000 [00:04<03:23, 14.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   1%|▏         | 42/3000 [00:05<06:45,  7.29it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   1%|▏         | 44/3000 [00:05<06:40,  7.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   2%|▏         | 51/3000 [00:06<04:22, 11.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   2%|▏         | 56/3000 [00:06<03:50, 12.74it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   2%|▏         | 58/3000 [00:06<04:18, 11.37it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   2%|▏         | 60/3000 [00:07<05:15,  9.32it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   2%|▏         | 62/3000 [00:07<05:39,  8.66it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   2%|▏         | 65/3000 [00:07<06:05,  8.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   2%|▏         | 68/3000 [00:08<05:04,  9.61it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   2%|▏         | 72/3000 [00:08<03:20, 14.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   3%|▎         | 76/3000 [00:08<03:37, 13.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   3%|▎         | 78/3000 [00:08<04:26, 10.96it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   3%|▎         | 82/3000 [00:08<03:42, 13.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   3%|▎         | 84/3000 [00:09<05:31,  8.80it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   3%|▎         | 88/3000 [00:09<05:15,  9.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   3%|▎         | 90/3000 [00:09<04:57,  9.77it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   3%|▎         | 94/3000 [00:10<04:45, 10.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   3%|▎         | 97/3000 [00:10<04:14, 11.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   3%|▎         | 99/3000 [00:10<04:49, 10.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   3%|▎         | 102/3000 [00:11<04:08, 11.67it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   3%|▎         | 104/3000 [00:11<04:58,  9.71it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   4%|▎         | 109/3000 [00:11<04:10, 11.55it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   4%|▎         | 111/3000 [00:12<05:06,  9.42it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   4%|▍         | 116/3000 [00:12<03:47, 12.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   4%|▍         | 119/3000 [00:12<03:56, 12.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   4%|▍         | 123/3000 [00:13<04:49,  9.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   4%|▍         | 125/3000 [00:13<04:29, 10.67it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   4%|▍         | 131/3000 [00:14<04:49,  9.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   4%|▍         | 133/3000 [00:14<04:34, 10.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   5%|▍         | 138/3000 [00:14<03:35, 13.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   5%|▍         | 140/3000 [00:14<03:35, 13.27it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   5%|▍         | 144/3000 [00:15<05:14,  9.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   5%|▍         | 146/3000 [00:15<05:16,  9.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   5%|▍         | 148/3000 [00:15<04:29, 10.57it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   5%|▌         | 155/3000 [00:16<03:05, 15.38it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   5%|▌         | 160/3000 [00:16<03:34, 13.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   5%|▌         | 162/3000 [00:16<04:12, 11.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   5%|▌         | 164/3000 [00:16<04:37, 10.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   6%|▌         | 166/3000 [00:17<06:16,  7.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   6%|▌         | 173/3000 [00:17<04:22, 10.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   6%|▌         | 175/3000 [00:18<04:21, 10.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   6%|▌         | 180/3000 [00:18<03:40, 12.78it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   6%|▌         | 182/3000 [00:18<04:53,  9.60it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   6%|▌         | 184/3000 [00:19<05:18,  8.84it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   6%|▋         | 189/3000 [00:19<05:19,  8.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   6%|▋         | 192/3000 [00:19<04:05, 11.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   7%|▋         | 196/3000 [00:20<04:15, 11.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   7%|▋         | 201/3000 [00:20<03:34, 13.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   7%|▋         | 203/3000 [00:20<04:51,  9.61it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   7%|▋         | 207/3000 [00:21<05:40,  8.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   7%|▋         | 209/3000 [00:21<05:06,  9.09it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   7%|▋         | 213/3000 [00:21<03:59, 11.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   7%|▋         | 217/3000 [00:22<04:18, 10.78it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   7%|▋         | 221/3000 [00:22<03:09, 14.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   7%|▋         | 223/3000 [00:22<03:16, 14.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   8%|▊         | 225/3000 [00:23<06:04,  7.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   8%|▊         | 227/3000 [00:23<05:20,  8.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   8%|▊         | 232/3000 [00:23<04:09, 11.09it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   8%|▊         | 234/3000 [00:24<05:43,  8.05it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   8%|▊         | 239/3000 [00:24<03:56, 11.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   8%|▊         | 242/3000 [00:24<03:14, 14.16it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   8%|▊         | 244/3000 [00:24<04:12, 10.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   8%|▊         | 248/3000 [00:25<04:12, 10.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   8%|▊         | 251/3000 [00:25<03:39, 12.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   8%|▊         | 255/3000 [00:26<05:30,  8.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   9%|▊         | 257/3000 [00:26<04:47,  9.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   9%|▉         | 263/3000 [00:26<02:48, 16.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   9%|▉         | 267/3000 [01:00<2:22:21,  3.13s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   9%|▉         | 269/3000 [01:00<1:45:39,  2.32s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   9%|▉         | 272/3000 [01:01<1:07:53,  1.49s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   9%|▉         | 278/3000 [01:02<32:52,  1.38it/s]  "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  10%|▉         | 285/3000 [01:02<14:59,  3.02it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  10%|▉         | 290/3000 [01:03<13:01,  3.47it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  10%|▉         | 294/3000 [01:03<08:31,  5.29it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  10%|█         | 303/3000 [01:04<04:26, 10.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  10%|█         | 306/3000 [01:04<04:03, 11.07it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  10%|█         | 309/3000 [01:05<06:32,  6.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  10%|█         | 311/3000 [01:05<06:12,  7.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  11%|█         | 318/3000 [01:05<03:55, 11.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  11%|█         | 323/3000 [01:06<03:23, 13.17it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  11%|█         | 325/3000 [01:06<03:39, 12.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  11%|█         | 327/3000 [01:06<03:42, 12.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  11%|█         | 329/3000 [01:07<05:47,  7.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  11%|█         | 331/3000 [01:07<04:56,  9.02it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  11%|█         | 335/3000 [01:07<04:30,  9.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  11%|█         | 337/3000 [01:08<05:24,  8.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  11%|█▏        | 341/3000 [01:08<05:17,  8.38it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  11%|█▏        | 344/3000 [01:08<04:09, 10.63it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  12%|█▏        | 346/3000 [01:08<04:23, 10.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  12%|█▏        | 348/3000 [01:09<04:42,  9.38it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  12%|█▏        | 353/3000 [01:09<03:57, 11.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  12%|█▏        | 357/3000 [01:09<02:50, 15.48it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  12%|█▏        | 360/3000 [01:10<04:27,  9.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  12%|█▏        | 364/3000 [01:10<04:02, 10.86it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  12%|█▏        | 366/3000 [01:10<04:18, 10.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  12%|█▏        | 368/3000 [01:11<04:42,  9.32it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  12%|█▏        | 373/3000 [01:11<03:26, 12.71it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  12%|█▎        | 375/3000 [01:11<03:24, 12.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  13%|█▎        | 379/3000 [01:12<04:34,  9.55it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  13%|█▎        | 383/3000 [01:12<03:54, 11.14it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  13%|█▎        | 385/3000 [01:12<03:36, 12.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  13%|█▎        | 387/3000 [01:12<03:37, 12.04it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  13%|█▎        | 391/3000 [01:13<04:23,  9.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  13%|█▎        | 394/3000 [01:13<03:36, 12.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  13%|█▎        | 396/3000 [01:13<04:39,  9.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  13%|█▎        | 401/3000 [01:14<03:48, 11.38it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  13%|█▎        | 403/3000 [01:14<03:53, 11.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  14%|█▎        | 405/3000 [01:14<04:28,  9.66it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  14%|█▎        | 409/3000 [01:14<03:54, 11.06it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  14%|█▎        | 411/3000 [01:15<03:45, 11.47it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  14%|█▍        | 415/3000 [01:15<03:55, 10.97it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  14%|█▍        | 418/3000 [01:15<02:59, 14.38it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  14%|█▍        | 420/3000 [01:15<04:09, 10.36it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  14%|█▍        | 422/3000 [01:16<04:52,  8.80it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  14%|█▍        | 424/3000 [01:16<05:21,  8.02it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  14%|█▍        | 426/3000 [01:16<05:21,  8.02it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303e135590>\n",
      "Evaluating workflow:  14%|█▍        | 427/3000 [01:17<09:20,  4.59it/s]Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bf1cde0>, 1554091.972634176)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303e137790>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303e2bead0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bf1f000>, 1554091.91433132)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303e0c3f10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303dfc0210>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048b25860>, 1554092.345626401)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303dfc0110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303e4e2110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303df442d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303df89890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153105034ad0>, 1554093.641959824)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303de52b90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15305667a040>, 1554091.681494142)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303df3fb90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048a4b230>, 1554092.043674912)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303df89750>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303de66ad0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530488b9b70>, 1554093.69505438)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303e0c3f90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303de93c10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153105034c90>, 1554093.645640692)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303de937d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303dd2b310>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531043d29e0>, 1554094.522294477)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303dd2b5d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303dd18f90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531043d1010>, 1554094.281936421)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303dd19550>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303debab50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153104f15240>, 1554095.458700561)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303de936d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303dc52f50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048a49cc0>, 1554096.244210982)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303dc52f10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303dd7ce50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153104f14ec0>, 1554094.623121296)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303dd7ce10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303db75c90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153104f16a50>, 1554097.448639054)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303db76110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303dbc5e50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153105036c80>, 1554097.826258738)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303dbc5dd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303d9ee7d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153104f16ba0>, 1554097.814413968)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303d9ee650>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303dd2a710>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303dce20d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303dc82610>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048b24bb0>, 1554096.084669131)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303dd29ad0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153103c00ec0>, 1554096.269156461)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303dce27d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153103c02350>, 1554096.298263383)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303dc82cd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303da0e610>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531044ea740>, 1554098.240839635)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303da62e50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303dad7990>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304be4b380>, 1554099.305250862)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303dad7350>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303db9a4d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303da2ef10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531038d6970>, 1554099.545018689)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303db9a5d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531044ea660>, 1554097.725399734)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303da2edd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303d95ebd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531038d6350>, 1554099.48504578)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303df1f390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303dab1750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153054e6b380>, 1554100.360163682)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303d86ff50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303dba04d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153054f7fc40>, 1554101.069422397)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303dbf3b50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303d8de890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531044eac10>, 1554101.088490158)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303d8deb50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303d80d010>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531044e88a0>, 1554101.794170597)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303d995fd0>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303d7adbd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304be4b000>, 1554101.916700933)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303d7addd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303d6fc250>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153054e6a970>, 1554102.240922153)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303d6fd1d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303d6187d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153054d6c2f0>, 1554102.683127121)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303d612550>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303d672e50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303ef01b00>, 1554102.935212125)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303d672590>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303d419f10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303d56b2a0>, 1554104.825258693)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303d41a250>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303d69b950>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303d549890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303d7700d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531044e8910>, 1554105.343728323)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303d69acd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531044ea200>, 1554103.722415209)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303d82cbd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303d5683d0>, 1554103.572546135)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303d770a90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303d457cd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303d4514e0>, 1554105.589374387)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303d457710>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303d301c50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303d453bd0>, 1554105.917774201)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303d5bce90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303d65a510>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303d354f30>, 1554106.973216099)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303d65b650>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303d209ad0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303d56bcb0>, 1554107.03853068)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303d303210>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303d3d2e50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303d24def0>, 1554107.355689294)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303d3d3a10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303d126450>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303d12ec10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303d24ff50>, 1554108.754199748)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303d12e690>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303d2948d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303d14b450>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303d1403d0>, 1554108.661955702)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303d2944d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303d140910>, 1554108.670782145)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303d14b8d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303d169a90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153054d6e190>, 1554108.632713364)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303d169950>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303d2049d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303d1b43d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303d140ec0>, 1554109.204199102)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303d2466d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303d141b70>, 1554108.663443039)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303d1b4a10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303d1c06d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303d142200>, 1554108.737809784)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303d1b7a90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303d181a50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303d1d7490>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303d142820>, 1554108.658893842)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303d180c90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303d1429e0>, 1554108.742158462)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303d1d7350>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303d0028d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303d143380>, 1554109.412574411)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303d6dae50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303d0088d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303d141860>, 1554109.035928497)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303d008a90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303d6da190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303d143e00>, 1554109.305804731)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303d023a10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303d05c310>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303d0389f0>, 1554109.148262255)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303d05c950>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303d2b7190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303d074950>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303d038d70>, 1554109.356962363)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303d00b150>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303d4508a0>, 1554109.525938208)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303d074810>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303d021050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303d142040>, 1554110.491358192)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303d098690>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303e02c0d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303e026590>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303e04bd10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303e073310>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303e1298d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303e087390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303e0a0d10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303e0b7f10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303e16dc50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303defc7d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303e0d5f50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303df50cd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303df53110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303df71ed0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303dfa5450>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303e0254d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303dfdd410>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303ddf3650>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303dfae650>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303de1c950>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303de37690>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303defe250>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303de90410>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303deaa550>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303dec1410>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303df73110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303dd26590>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303dd29110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303dd48e90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303dd5bc50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303dcf5f10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303dd05210>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303dd4b9d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303dd8f950>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303ddcaad0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303e3d9bd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303dea9050>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303dc1ac50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303dc33510>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303dcf5e50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303dc53fd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303dc79ed0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303dc9ad90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303dcc4410>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303dcd7750>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303db0d110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303db1ab50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303db36550>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303db1ea10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303dc10d90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303dc11ed0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303db64c90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303db5d350>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303dc83b90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303df3da10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303dbd1b10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303da49b50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303da60a10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303da7b450>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303da9d4d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303db1efd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303dac6c10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303db0d290>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303dae66d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303d8f5e10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303d91d790>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303d93a3d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303dbbca10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303d97c750>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303da49750>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303d9be3d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303d9d3ad0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303d9578d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303d7faad0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303d7ffc90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303d819650>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303d82e150>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303d8418d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303d91d4d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303d87d8d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303d894ed0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303d8a96d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303d8be9d0>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303d7119d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303d8bb110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303d72f690>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303d7572d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303d7954d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303d79fc10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303d7ca0d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303d780950>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303d62c110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303d6592d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303d712ed0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303d69ac10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303d6b5710>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303d9ab810>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303d797c10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303d79cb10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303d60fed0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303d62d790>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303d68e6d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303d5166d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303d8a8610>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303d5848d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303d58ef10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303d3edc90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303d65af50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303d40fed0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303d432f50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303d54ad10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303d5bcbd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303d5d1050>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303d52de90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303d489b10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303d4a3a50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303d4b5d10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303d4c7a90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303d4e04d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303d4cfdd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303d326e50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303d34ab90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303d35bf90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303d433cd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303d3a8350>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303d319350>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303d3aa490>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303d4a8c90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303d204050>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303d48aad0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303d23a710>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303d245ad0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303d2786d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303d4e2590>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303d2a1dd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303d2b6f50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303d2cda90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303d2e4550>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303d0efd50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303d452c80>, 1554108.734030687)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303d126e50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303d24eac0>, 1554108.378340552)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303d2e4c90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303d453c40>, 1554108.375877419)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303d0efc50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303d0391d0>, 1554109.418318701)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303d020b10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530489757f0>, 1554090.512810382)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303e02c090>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048a4a890>, 1554090.707756715)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303e026610>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048a49400>, 1554090.834344382)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303e04bbd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530489752b0>, 1554091.099127492)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303e1654d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd8d7c40>, 1554091.336730105)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303e12aa10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056678750>, 1554091.545992839)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303e087790>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048975240>, 1554091.968259872)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303e0a0b10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048a49320>, 1554092.207925656)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303e0b7e10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530488b9710>, 1554092.381397013)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303e0b8090>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bf1d240>, 1554091.994133963)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303e167190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bf1c6e0>, 1554091.816590338)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303e0d4890>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b45c910>, 1554092.423442388)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303e1d04d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048b25710>, 1554092.605468164)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303df52d50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048b279a0>, 1554092.399770955)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303df71d90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048b27d90>, 1554092.199704459)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303dfa5310>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048b24750>, 1554092.23180914)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303df59390>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048b241a0>, 1554092.789172268)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303dfdd750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530489758d0>, 1554092.926454399)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303ddf1310>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048b26d60>, 1554092.647611833)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303dfaddd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048b27cb0>, 1554093.008925793)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303de1c810>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bf1f4d0>, 1554093.414278788)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303de37510>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531050352b0>, 1554093.896131957)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303defded0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153105036ba0>, 1554094.015752584)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303df8add0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bf1cb40>, 1554093.623220762)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303deaa410>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153105037700>, 1554093.814816321)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303e0d7b90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531043d3f50>, 1554094.508603582)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303df72750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531043d2900>, 1554094.252427431)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303dd26a90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531043d0980>, 1554094.725527777)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303dd29810>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048b24130>, 1554094.317298102)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303dd45890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531043d3af0>, 1554094.342894005)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303dd5bb10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530488ba3c0>, 1554094.151243072)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303dcf5d10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531043d2120>, 1554094.375821501)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303dd05190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530489772a0>, 1554095.107599582)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303dd4b890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153104f15470>, 1554094.995091271)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303dd8f810>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153104f15710>, 1554095.223828056)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303ddc8910>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153104f17a80>, 1554095.611501306)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303dbf3f10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153104f169e0>, 1554095.430157611)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303de92a10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153104f14d70>, 1554095.449691866)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303dc1ab10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bf1fbd0>, 1554096.188221105)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303dc33390>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153103785d30>, 1554095.956125929)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303dcf68d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153103c02a50>, 1554095.796874989)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303dc53990>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153103c00130>, 1554096.210231489)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303dc79cd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153103c03bd0>, 1554095.896431896)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303dc9ac10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bf1d6a0>, 1554096.112185047)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303dcc4b50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bf1e510>, 1554096.5281461)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303dcd7610>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153103d078c0>, 1554097.189269116)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303db0cf50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153103d05b00>, 1554096.710482775)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303db1ae90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153103d065f0>, 1554096.827067366)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303db36c10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153103d04590>, 1554096.574282059)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303db1ec50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153103d05940>, 1554096.957311897)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303dc11010>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153103d06580>, 1554097.386487704)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303dbf3e90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153103d05da0>, 1554097.413506128)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303dbf4d90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153103d04d70>, 1554096.905037163)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303db5d4d0>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153104f14f30>, 1554097.912042669)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303da02e90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531044e8440>, 1554098.017697141)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303da1fa90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531044e89f0>, 1554098.010169743)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303dbd20d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153103c02f20>, 1554098.316496768)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303da49950>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531044ea040>, 1554098.158697797)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303da610d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048b27460>, 1554098.380820764)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303da7b310>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304be4b8c0>, 1554098.152165864)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303da9e3d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304be4a040>, 1554098.741326765)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303db1ff10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304be498d0>, 1554098.512260111)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303decadd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304be48980>, 1554098.553190644)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303db0e190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304be4a190>, 1554099.080669313)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303dae7f10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304be48590>, 1554098.925230025)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303d8f7c90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304be4a890>, 1554098.822898856)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303d91d590>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153103d04b40>, 1554099.263178948)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303d93a950>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531038d6270>, 1554099.766440048)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303d973fd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531038d5010>, 1554099.72033242)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303de6c710>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531044e92b0>, 1554099.651530077)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303d93ae50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531038d5e80>, 1554099.599427697)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303d97df10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531038d7af0>, 1554099.633406749)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303d9d3ed0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153054e682f0>, 1554100.025130005)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303d957a50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153054e69fd0>, 1554100.043756464)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303d7fa8d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153054e695c0>, 1554100.054847725)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303d9aad10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531038d5320>, 1554099.885052793)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303d819610>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153054e6ba80>, 1554099.96789631)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303d82e810>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153054e69400>, 1554100.741002605)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303d842090>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531043d2a50>, 1554100.210509492)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303d91ea90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153103c03a10>, 1554100.736055145)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303d87d790>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153103c03d90>, 1554100.586922226)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303d895410>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153054e69630>, 1554100.516385152)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303d8a94d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153054f7cb40>, 1554101.287164712)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303db61390>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153054e69b70>, 1554101.177065169)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303d7117d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153054f7f310>, 1554101.266009147)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303d8bb310>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153054f7c360>, 1554101.679863865)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303d72f510>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531044eb150>, 1554101.752899912)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303d757190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153054f7f2a0>, 1554101.592431359)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303d8d7090>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153054d6dbe0>, 1554102.096952585)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303d79fad0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153054d6d2b0>, 1554102.180859801)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303d7ca7d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531044e9710>, 1554101.611733988)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303d780750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153054d6c0c0>, 1554104.080342543)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303d62eb90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153054d6cbb0>, 1554102.915284834)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303d659190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153054d6d390>, 1554103.128536521)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303d712b10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303ef02200>, 1554103.388968522)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303d69b310>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153054f7d5c0>, 1554103.18025786)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303d90dcd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303ef016a0>, 1554103.639257304)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303d757610>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153054f7e740>, 1554103.628884218)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303d797690>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303ef02890>, 1554103.601512982)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303d4fbb50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153054e6b1c0>, 1554102.067870957)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303d60d890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153054e6a3c0>, 1554102.587859715)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303d72dcd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531044ebd20>, 1554103.248967571)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303d68df90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303ef02cf0>, 1554103.890773932)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303d6b5cd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153054d6de80>, 1554104.732453314)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303d573f90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303d568980>, 1554104.182371538)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303d584f90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303ef03f50>, 1554104.029265362)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303d58ed10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303d5698d0>, 1554104.856709153)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303d3eda90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303d56a7b0>, 1554104.846315377)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303d5495d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303d56af20>, 1554105.42798333)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303d40fdd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303d56b7e0>, 1554105.182329162)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303d433290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303ef039a0>, 1554104.025341991)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303d54a9d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153054d6fee0>, 1554104.259274094)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303d874c10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153054d6cd00>, 1554104.700336435)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303d5d1790>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303d4500c0>, 1554105.215478872)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303d447bd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303d451240>, 1554105.654626465)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303d48a250>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303d4516a0>, 1554105.436171917)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303d4a3910>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303d451d30>, 1554105.502786113)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303d4b63d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153054f7ef90>, 1554105.663854206)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303d4c7950>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531038d4e50>, 1554106.10108746)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303d4e0250>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303d4523c0>, 1554105.667083691)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303d566050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153054e690f0>, 1554105.983414475)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303d326d10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303d354210>, 1554106.310309674)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303d34b250>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303d3550f0>, 1554106.436782923)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303d35b8d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303d354bb0>, 1554106.585691276)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303d433e10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303d355a90>, 1554106.555204198)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303d3a8a10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303d4533f0>, 1554105.539949472)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303dc1b8d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303d355c50>, 1554106.610095846)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303d3abf50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303d356ac0>, 1554107.147025086)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303d4a8d90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303d356cf0>, 1554107.263134644)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303d204350>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153054f7df60>, 1554106.922595635)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303d489f50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303d24c210>, 1554107.565644631)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303d81b310>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303d24c3d0>, 1554106.918506138)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303d2458d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303d24cfa0>, 1554107.052835967)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303d2784d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303d24d550>, 1554107.714084394)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303d4e2a50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303d24dd30>, 1554107.549747264)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303d2a24d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303d24e0b0>, 1554107.609105073)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303d2b6e10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303d24e510>, 1554107.494107083)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303d2cd890>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303d5f53d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303d5266d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531043d2c80>, 1554101.940771075)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303d5f5290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303ef03230>, 1554103.706261708)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303d527910>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303d5499d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303ceff250>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303cef9a90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303d0cca90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303d045ed0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056d69400>, 1554110.917131025)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303d0cc8d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303d143230>, 1554110.848483677)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303d0d64d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303d5bf790>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd5da660>, 1554111.002327204)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303d5bd4d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303d7c8b50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303d8dfd50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b855400>, 1554112.60563758)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303d726050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b856190>, 1554111.531693578)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303d8de690>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303fbfab10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303dbbde50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b854130>, 1554111.488216996)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303fbf8ad0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b855470>, 1554111.451807019)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303d964cd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303d643c50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b855d30>, 1554111.128458651)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303d677790>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303dbf54d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303dd0ec10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b856660>, 1554111.262739601)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303cf81410>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fc7cba10>, 1554112.23112067)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303dd0d010>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303ceed790>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303df1c2d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fc7caa50>, 1554111.908725488)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303cf84e90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b855fd0>, 1554111.812997006)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303dbf4710>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303dfb74d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303cf1fe50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303cf23e50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303cf51ad0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303cf68450>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303cf53550>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056d6ac10>, 1554110.989331764)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303cf3f9d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303d24e6d0>, 1554110.522373381)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303cefefd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056d68bb0>, 1554111.151435072)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303cef9650>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303ceec650>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303d0c0bd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303cf82690>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056d68e50>, 1554110.453860903)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303ceeef50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153105245f60>, 1554111.964908948)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303dfb5d50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd2210f0>, 1554110.378482714)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303cf1fcd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048f26b30>, 1554109.843113066)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303d173e90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303d451400>, 1554110.277796671)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303cf9b7d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056d68980>, 1554110.044647533)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303cf83a50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056d6bb60>, 1554110.341786851)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303cf50290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303d56a660>, 1554110.698890195)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303d0c0090>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531043d1da0>, 1554110.778968028)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303cf81ad0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303e19dc10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303d24d5c0>, 1554112.450556769)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303e19fc90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303e1d72d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303e008550>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056dd9860>, 1554112.340513549)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303e1d4d90>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fc7cb000>, 1554112.373440121)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303e00a190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303e1d0f90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056d69240>, 1554112.534440877)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303e1d1310>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303d0463d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd6b5b00>, 1554113.071426972)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303d0452d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303df5a990>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303fcd5a90>, 1554112.794545486)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303df59550>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303e04b490>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303e292b10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303d24d2b0>, 1554113.060895607)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303e049410>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd1db150>, 1554113.148991702)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303e2b3c50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303e17a910>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303e277fd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd1da430>, 1554112.834613505)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303e17bf90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b8554e0>, 1554113.124710862)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303e274110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303db66590>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fe436d60>, 1554113.067326548)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303d6617d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303e3ce2d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fc0a3150>, 1554113.27138227)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303e3ce810>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303d9cadd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303e137bd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303d24fa10>, 1554113.035949615)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303d9cb5d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd5070e0>, 1554113.430734693)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303e134b10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303e3acc50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fc7c9080>, 1554112.932542159)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303e3adfd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303e12b7d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303e36d990>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd507af0>, 1554113.582329379)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303e12bb90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd5051d0>, 1554113.95113774)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303e36d950>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303e331510>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057fdd550>, 1554114.153866456)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303e331050>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303e3124d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fcee0ec0>, 1554114.135886273)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303e311d50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303e47e210>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fc7cbee0>, 1554114.009964187)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303e47f110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303e359550>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303e486810>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fcee14e0>, 1554114.92032912)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303e3580d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153105246dd0>, 1554114.73557625)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303e485950>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303e473490>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b658d00>, 1554115.15496595)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303e471090>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303cf99550>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fc520d70>, 1554114.624636274)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303e43d150>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303d61a390>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fe434210>, 1554114.615419822)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303e051c50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303e2a1310>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b65af90>, 1554114.915372026)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303e411e10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303e5c4cd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303e5a9590>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b65a270>, 1554114.806409734)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303e5c5850>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303d24f310>, 1554115.160344368)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303e5a9650>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303e57a290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b659240>, 1554114.698803824)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303e579e50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303e69b590>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303e547410>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fca3c2f0>, 1554114.827715706)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303e69a390>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056d6a5f0>, 1554115.133527421)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303e546f90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303e54d8d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303e514590>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b65a120>, 1554115.000401301)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303e56f690>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048b249f0>, 1554114.889736959)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303e5143d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303e5198d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303e61bfd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd5047c0>, 1554115.774626399)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303e135890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b5583d0>, 1554115.760155832)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303e274bd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303e631dd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303de91e10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057fdf7e0>, 1554116.058860492)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303e6317d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b559780>, 1554115.519633319)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303de90e90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303e6308d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153055ff9650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b558210>, 1554116.940901777)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303e31a390>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b55ae40>, 1554115.873754137)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153055ffbed0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303e136d10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303e1d5890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057e93d90>, 1554117.072292121)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153055f22d10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057e907c0>, 1554116.229685216)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303e1d70d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303e794290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b55b540>, 1554116.933741514)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303e795890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303e60cbd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303e7746d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057e93000>, 1554116.234246409)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303e60ed10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303d1407c0>, 1554116.43422048)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303e7751d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303e7d84d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd347230>, 1554116.618983012)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303e7da150>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303e79ea50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153055d53350>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303e6df890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b559cc0>, 1554114.978002189)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303e6dfc50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b658f30>, 1554117.406782068)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303e79cfd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303d4512b0>, 1554116.699758257)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153055d51490>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153055f9b850>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bc28050>, 1554117.16891855)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153055f9b290>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153055e11110>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fca3fbd0>, 1554117.051398681)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153055e10c10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530561502d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153055e401d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bc2a430>, 1554117.374145036)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153056153c90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bc28d70>, 1554117.191325729)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153055e42290>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303e6184d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bc288a0>, 1554117.558096549)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303e618410>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303d2284d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153055ad66d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bc29710>, 1554117.675939251)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303e631410>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153104f16120>, 1554117.442825527)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153055ad7490>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153055b828d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153104f17cb0>, 1554117.600759748)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153055b834d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153055bb4390>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531079baba0>, 1554117.882572141)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153055bb6ed0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303e775e50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303e4cd1d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057fdc7c0>, 1554117.91574637)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153055a55210>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd220130>, 1554117.9957136)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153055ac9010>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530559653d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd223d90>, 1554118.326756177)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153055977250>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153055986c10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd1d9cc0>, 1554118.490109322)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530558aa950>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153055802ed0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153055d96f10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303e053350>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bc29f60>, 1554116.742178759)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303df5ad50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bc2aac0>, 1554116.840015972)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153055d975d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048b250f0>, 1554119.352737102)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153055803d50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153055550990>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303d357460>, 1554119.789457708)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153055550e90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153055aec0d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530428ea610>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153056141890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303e311110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530559b4490>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530557e4190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15305564a190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15305536a490>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530556f0090>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530556b8390>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b2601a0>, 1554118.758652542)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153055aef890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bc2bb60>, 1554118.591882564)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530428eadd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b2639a0>, 1554119.434286735)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153056169790>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b260670>, 1554118.975670686)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153055a01610>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057e92580>, 1554118.878983231)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530559b4510>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b263620>, 1554119.248633899)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530557e77d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057e90a60>, 1554119.373071306)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15305564b150>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fca3ce50>, 1554119.274955984)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153055369550>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bf1c050>, 1554119.490459721)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530556f17d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b55b700>, 1554119.366726425)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530428e85d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153055891650>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530558ea5d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b262510>, 1554119.025999129)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303cf982d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b263c40>, 1554118.705692929)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530558e8f90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b857d20>, 1554851.593534856)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303e2bd610>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b855be0>, 1554851.823637892)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303e10b550>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303d565190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303d53fb90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b855a90>, 1554851.492342883)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303d565c90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303d221d90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303e1f39d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303e38ad90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303e375dd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303e58f6d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303e54cc50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303e5a82d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303e75a3d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303ef3b910>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b854ec0>, 1554851.324674206)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153048e00e90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fc3ae2e0>, 1554851.450584895)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304902a490>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b8578c0>, 1554851.557928329)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303d53fc50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b856040>, 1554851.310913754)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303d223390>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fcee1ef0>, 1554851.549640473)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303e1f0190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b856ba0>, 1554851.734372518)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303e3ccd10>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b855710>, 1554851.466513681)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303e421f50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b8567b0>, 1554851.581976615)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303e49a890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fcee0280>, 1554852.889807735)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303e54ced0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b856740>, 1554851.207516158)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303e652a50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b8571c0>, 1554850.831100128)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303e75a5d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b854750>, 1554851.380946561)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303ef3a050>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304929cc10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530493dd850>\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153049400f90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b856e40>, 1554851.433235819)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153049403f50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153049574a90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b857c40>, 1554851.578128858)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153049775a50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b855390>, 1554851.471330341)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530492b81d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153105036d60>, 1554851.194062428)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530492c0350>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fcee06e0>, 1554851.387970733)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304963dbd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15305665ce90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd6b4ad0>, 1554853.211345545)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303e5a2dd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304b5c4ad0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304b806450>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd6b5780>, 1554853.150053928)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304b5c7090>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304ba86f10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fc0a2c10>, 1554853.001372755)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304ba84bd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304b3a5650>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fc4657d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fc0a3bd0>, 1554853.34147536)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304b4b1fd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153104f17230>, 1554853.249289803)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fc466590>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304b34f7d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531052a8390>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153105036120>, 1554853.420228827)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304b806790>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b5598d0>, 1554853.354973069)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304b51b9d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b559b70>, 1554853.69445414)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fce13e10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fcab8c50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b558280>, 1554853.243862808)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303e291650>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303e33ee90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531079bb3f0>, 1554853.689678486)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303e33d110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153056eb03d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153057a32950>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b2612b0>, 1554853.865941767)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304be93890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b261630>, 1554853.702213026)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153057a33a10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fc5ab790>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153054f7e5f0>, 1554854.62772521)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fc5a8490>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fc6e3190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fd4a4290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b261e80>, 1554855.131015817)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fc6e25d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd3446e0>, 1554854.858154824)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303e669550>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fc661250>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303f3f0e50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fcde5610>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fc31ca90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153056e14d10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153056dd6850>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303f1d7050>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15305794a950>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531079b8440>, 1554853.299222271)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153057952d50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b263540>, 1554853.356901573)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153049794410>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b558980>, 1554853.464900997)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fcde4a50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153054f7d080>, 1554854.790329377)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fc663a10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd223000>, 1554855.193241686)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303f3f0ed0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531044e9240>, 1554853.205326305)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153056e16790>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303d3569e0>, 1554853.610580001)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153056dd6610>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b262ac0>, 1554853.425641084)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fc663610>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fcd438d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b260a60>, 1554852.970744103)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153101f76750>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fcde68d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153054f7f8c0>, 1554855.999874371)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153105daea10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303f4589d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303fa91a90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd2230e0>, 1554855.114223258)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fcab9150>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd220d70>, 1554855.433245377)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303fa91d10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303faa86d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303fb96cd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531038d7a80>, 1554855.415545354)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303faa96d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd223620>, 1554856.142995222)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303fb96b50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153042f69dd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153105246270>, 1554855.509778374)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15310791b710>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153042e2dfd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fc634350>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fca3dfd0>, 1554855.504787287)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fc637750>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153042bce050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b558bb0>, 1554855.796837296)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153042bccf50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153048acd210>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bc2b2a0>, 1554855.8310338)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153048acd090>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fc85ded0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303d73ad90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bc2aeb0>, 1554855.635827262)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fc85fbd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bc28750>, 1554855.675030977)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303d73b450>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153042e81850>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bc29860>, 1554856.070054756)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153042e825d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fc28aa10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bc28c20>, 1554856.40141621)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fc28a050>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fd9d7c10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fc72df50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fca3f620>, 1554855.555734695)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153042e2cc90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304b281f50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd223af0>, 1554854.975510826)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304b2823d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531038d7460>, 1554856.456574682)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fd9d7ad0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd221630>, 1554855.518562032)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fc72d950>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531051502d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153054f7d940>, 1554856.678920457)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153105150e90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153105152e90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b65bd20>, 1554856.998992118)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531051527d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fdc79350>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530ff5c8fd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd507150>, 1554856.874968432)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fdc79090>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303f254250>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b260ec0>, 1554857.525376028)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303f2548d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fdc78290>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530549b3910>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303d357d20>, 1554857.328837666)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fcd9f6d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fc5220b0>, 1554857.239424212)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530549b0a10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303fb47250>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057fdf5b0>, 1554857.497866177)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303fb47110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303cf48c90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fdc254d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057fdd8d0>, 1554857.366372078)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fc9a3490>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057fdc3d0>, 1554857.714003837)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fdc25490>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303f2567d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153035970390>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057fdf310>, 1554857.107450274)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303f256410>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304be496a0>, 1554857.393233529)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530359709d0>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153035973650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048b25b00>, 1554857.433362627)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153035973450>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153034df05d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153034dbbad0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bc29c50>, 1554857.928767953)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153034df1c90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fe437b60>, 1554857.456700649)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153034dbb990>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153034d82ed0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd504d70>, 1554857.657695683)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530ff5c8e90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057fdc280>, 1554858.163839108)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153034d82cd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153034dd54d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057e930e0>, 1554858.321761258)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153034dd5ad0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303f364790>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fda24a90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057e90050>, 1554858.091278973)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fda244d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303f529190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bc28980>, 1554858.227240403)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303faa9e10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153034e49bd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153034ff6fd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bf1ce50>, 1554858.437360808)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303f256850>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bf1dc50>, 1554858.745792719)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153034ff5890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153034fbded0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153034f4d4d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048975ef0>, 1554859.343579987)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153034f4e790>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303f254150>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd220d00>, 1554859.053610869)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303f254cd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153048acf2d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530350c2310>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048977c40>, 1554859.166956854)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153054968d10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048974130>, 1554859.265675128)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530350c2110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153034fdbc50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530350511d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048a483d0>, 1554859.530968041)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153034fdbd10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048a4ba10>, 1554858.982760346)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153035050fd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153034f84f90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530352e4910>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153034d11650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b260fa0>, 1554858.338119557)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153034ebff90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b6581a0>, 1554858.845987905)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153034fa2110>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048a49e80>, 1554859.419110643)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153034f84f10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd220c20>, 1554859.59131317)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530352e4710>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057e91240>, 1554857.821831082)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153034d11490>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530352afd50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048a48e50>, 1554859.796450574)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530352afc50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153034f333d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303ef02660>, 1554860.737054908)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153034f334d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153034e9e310>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530353e8cd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048a4ad60>, 1554859.731710352)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530352aecd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048a49d30>, 1554860.052795186)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530353e8b90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530353b3e10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530353957d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057e911d0>, 1554859.720036465)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530353b3c90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530488ba900>, 1554860.260445254)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530353955d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153035397490>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153035325cd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048a4b540>, 1554860.217152233)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fd60d250>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530488bbd20>, 1554859.933823699)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153035325b90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530352045d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530354b4c90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fe437000>, 1554860.383084125)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530353ce6d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303547d4d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303540c810>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530488bb7e0>, 1554860.512914305)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303547d890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b659940>, 1554860.688175421)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303540c610>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530355f3e10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530488b83d0>, 1554861.752194002)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530355f3cd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530355b49d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056679320>, 1554860.959544055)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530355b50d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303557d3d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048a4b5b0>, 1554860.199898761)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530354b4b50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056679780>, 1554861.04111544)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303557d290>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153035545e10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15305667b460>, 1554861.08866769)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303598edd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153035599690>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530356f2350>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b260600>, 1554861.658485645)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153035599610>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531038d6890>, 1554861.607656745)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530356f2210>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530356804d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd507e00>, 1554861.050708636)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153035681e10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530356664d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15305667b1c0>, 1554861.662553501)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153035666390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530357dac50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b45d940>, 1554861.357698791)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530357da890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530357bd9d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b45c600>, 1554861.601738724)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530357be090>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530357a0150>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b45f7e0>, 1554861.747288297)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530357a04d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153034d114d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153035716590>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048a49b70>, 1554861.714210835)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530354b5890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b45d9b0>, 1554861.41265847)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153035716390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530358dd650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b45d8d0>, 1554861.978379933)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530358fadd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530358df850>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303588afd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b45eb30>, 1554862.138213804)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530358dfa50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b45f3f0>, 1554862.504688014)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303588add0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153035853710>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b45e350>, 1554862.326110907)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153035853550>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530358a5450>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b45df60>, 1554862.350680565)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303530aa10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530351fc250>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303518c2d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530488ba200>, 1554862.72166452)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530351fc110>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056679d30>, 1554862.661109041)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303518c790>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153035155d90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bf1c360>, 1554862.530735647)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153035155c50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530351e2a50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303511e190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531037844b0>, 1554863.172172788)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530357a33d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531037878c0>, 1554863.351908473)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303511e050>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303562ec50>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531037852b0>, 1554862.65497782)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303562e790>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153034c21910>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153103787a80>, 1554863.210803238)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153034c21a90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153034c28310>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153034c2f590>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048b26660>, 1554863.263279285)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153034c28350>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153103784d70>, 1554863.718700837)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153034c2f390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153034c33350>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153034c27b10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531037847c0>, 1554863.733752528)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153034c315d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153103787540>, 1554863.974630759)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153034c25a90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153034c1c7d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153034c48e10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153103787bd0>, 1554863.51239074)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153034c266d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531037879a0>, 1554863.867564957)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153034c49590>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153034c556d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530488b8130>, 1554863.803064733)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153034c55dd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530355d4150>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057f43230>, 1554863.458475278)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530355d55d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153034c4b9d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057f40360>, 1554863.822679396)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153034c5eed0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153034c65d90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057f42820>, 1554863.671296004)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303530a210>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153034c7a3d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153034c7d2d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd8d4ad0>, 1554864.539961668)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153034c65050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531037866d0>, 1554863.995855098)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153034c7d9d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153034c8c6d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b45d550>, 1554865.331111986)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153034c8a190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303518d6d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd8d6cf0>, 1554864.630782125)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303518d850>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153034c7f6d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153034c98210>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd8d6820>, 1554864.857614091)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153034c7f490>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530488b8c90>, 1554864.722938046)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153034c988d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153034c94390>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd8d4670>, 1554864.466925569)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153034c940d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153034ca2b90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd8d7e00>, 1554864.965384973)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153035396b90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153034cb6d10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153034c29cd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153034cc9e90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056679f60>, 1554864.904137608)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153034c2ad10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b45e820>, 1554865.277527539)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153034cca350>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153034cd4490>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531050f9da0>, 1554865.368941924)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153034cd4290>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153034cdcc90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd8d4360>, 1554865.138630561)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153034cdd110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153034ce1c90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd8d4a60>, 1554865.258925794)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153034cb6bd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531050fbbd0>, 1554865.37059985)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153034c566d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153034ce4350>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153103785320>, 1554865.289500789)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153034ce4250>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153034cf9a90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153034cd6650>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530340ffd90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531050fb7e0>, 1554865.786815791)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530358df750>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531050fb070>, 1554865.855885359)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530340ffc50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153034ce1090>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303410f090>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531050fb000>, 1554865.706219913)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153034ceb3d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd8d54e0>, 1554865.66108746)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303410ef50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153034ca04d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303d03a6d0>, 1554866.091936267)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530ff59ad50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153034c987d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303412cd50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057f41710>, 1554866.338516283)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153034124450>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd8d69e0>, 1554866.814096174)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530351e2dd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153034137d90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303d0388a0>, 1554866.327412942)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153034137c50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153034ca3050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153103787ee0>, 1554867.037872363)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153034ca2dd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530341477d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303d039da0>, 1554866.794088402)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530341458d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153034152110>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303d03af90>, 1554866.622482895)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153034152090>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153034162810>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153034cef8d0>\n",
      "Evaluating workflow:  14%|█▍        | 433/3000 [01:17<04:57,  8.62it/s]Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531050fba80>, 1554865.831282082)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153034cf9950>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531050f9470>, 1554865.320448825)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153034cef790>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd8d72a0>, 1554866.83744642)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153034162610>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530341501d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153034176c90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd8d7d20>, 1554866.828729711)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530341514d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd7d0210>, 1554867.234118215)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153034176b50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153034ccb250>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153034180c90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd7d32a0>, 1554867.415735698)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153034cca690>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd7d2eb0>, 1554867.624697306)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153034180a90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303415dad0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd7d0050>, 1554867.556731916)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303415d910>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303415f510>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd7d0c20>, 1554867.084900548)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303415f4d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530341a6150>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153034104d90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531050fba10>, 1554867.764095778)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530341a63d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd7d30e0>, 1554867.386369033)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153034104610>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303419d4d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530341b8890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057f43c40>, 1554867.704394519)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303419d610>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531050f8d00>, 1554867.638421296)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530341b8c50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530341bf4d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd7d1860>, 1554867.775476172)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530341bf9d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153034195950>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530341d66d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd42d160>, 1554868.126980628)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153034105f50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15305667ac80>, 1554868.238853412)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530341d7d10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530341dae50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd42f770>, 1554868.402686751)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530341dad10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530341cca90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153034193e90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531050fa2e0>, 1554868.407447483)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530341471d0>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd7d15c0>, 1554866.875015753)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153034193d90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530341f1c10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302dafcd90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd8d5080>, 1554869.136117288)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530341f1b90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531050f8a60>, 1554868.93869248)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302dafcc50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530341e7350>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302db0c510>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd8d7310>, 1554869.019044244)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530341e7590>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302db10710>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd759160>, 1554868.498814777)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153034c98cd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530341f4b10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd758b40>, 1554869.088510481)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302db23fd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302db29610>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fccb3bd0>, 1554869.167484066)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153034148f10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153034186390>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303d03a200>, 1554868.892564728)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153034186410>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302db41090>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302db59910>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd7d01a0>, 1554869.680524447)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302db41650>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302db69150>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd7d1be0>, 1554869.642288308)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302db68f50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302db17190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153100bc17f0>, 1554869.459537333)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153034c83b50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302db3f610>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302db81ed0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd221da0>, 1554869.842785084)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302db74f50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153100bc0520>, 1554869.625204375)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302db82610>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302db88a50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153100bc20b0>, 1554869.55199978)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302db88850>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302db4fe50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302db9fad0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd75b070>, 1554869.065540551)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302db0cbd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531050f83d0>, 1554869.441446667)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302db59750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153100bc08a0>, 1554869.654512125)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302db4ff50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd13fcb0>, 1554870.356725514)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302db9ff90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302dbb07d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303d03a040>, 1554870.006261478)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302db28d90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530341eb4d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd13db00>, 1554870.693987431)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303410efd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302db22f90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302dbd0450>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd13f150>, 1554870.649308794)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153034110490>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd13cf30>, 1554870.201356732)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302dbd0b10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302dbe7010>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fe39ff50>, 1554870.936768501)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302dbe6e10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153034185dd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302da016d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd75baf0>, 1554870.976143222)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153034186110>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fccb1240>, 1554870.814663051)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302da01590>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302dbd66d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302da0f510>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd75ac80>, 1554871.028905972)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302dbd6750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fe39eeb0>, 1554870.998177678)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302da0f3d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302da13610>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302da22390>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fe39fe00>, 1554871.011852299)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302da13b10>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302da41f90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302db54050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303f1efee0>, 1554871.923438096)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302db63e90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302db60950>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302da65d90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303f1eeeb0>, 1554871.501516591)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302da22190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303f1ed8d0>, 1554871.202449019)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302da45a50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303f1eee40>, 1554871.389367522)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302db60290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303f1ecd70>, 1554871.468126728)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302da65c50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302da738d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303f1eda90>, 1554871.76780306)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302da73790>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302da778d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303f1ec3d0>, 1554871.674531846)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302da75d50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302da52710>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302da95f10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303f1ee660>, 1554871.633619482)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302db96bd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302daa48d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302daa9b10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048f24440>, 1554872.413022688)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302daaa250>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302dac1010>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd758a60>, 1554872.552515307)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302dac2b50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302dbc04d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048f24670>, 1554872.764119929)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302da35a10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302dad66d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048f24c20>, 1554873.042575054)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302dad6690>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302da96b10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048f26120>, 1554872.67833451)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302da95d90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302db2a410>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d8f8b50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048f26a50>, 1554872.696044204)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302db4fe10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048f25da0>, 1554872.934696983)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d8f8a10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302da13f50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d9054d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048f27620>, 1554873.483516134)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302da10190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048f255c0>, 1554872.795124944)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d906950>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302da24ad0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302da396d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd13f070>, 1554871.627864564)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302da95d10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303f1ecb40>, 1554871.872009422)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302daa46d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048f24d70>, 1554872.975332689)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302da2a190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd7d0c90>, 1554871.644378315)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302da39090>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d8f6910>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d936690>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303f1ecfa0>, 1554873.315555865)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d8f6850>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d93d750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303fce6b30>, 1554873.720880741)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d93d550>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d954d50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303f1ee270>, 1554873.851267149)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d955250>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d92ce10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d964610>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303fce7ee0>, 1554873.477577291)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d92d310>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303fce7150>, 1554873.726973745)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d964410>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302dabd250>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d978a90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303fce7af0>, 1554873.58931116)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302dabd110>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056c537e0>, 1554873.590453429)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d978f50>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302da964d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d983590>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd13d550>, 1554873.86644809)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302da970d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048f25320>, 1554873.760061627)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d983450>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302daaac90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048f278c0>, 1554874.572004666)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302daaae90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302da775d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056ee7a10>, 1554874.490267543)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d9f03d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd13e660>, 1554873.574077782)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d936490>\n",
      "Evaluating workflow:  14%|█▍        | 435/3000 [01:17<04:29,  9.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  15%|█▍        | 440/3000 [01:18<03:24, 12.49it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  15%|█▍        | 445/3000 [01:19<05:52,  7.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  15%|█▌        | 452/3000 [01:19<03:53, 10.93it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  15%|█▌        | 456/3000 [01:19<03:41, 11.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  15%|█▌        | 462/3000 [01:20<02:39, 15.88it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  15%|█▌        | 464/3000 [01:20<05:08,  8.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  16%|█▌        | 469/3000 [01:21<04:01, 10.46it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  16%|█▌        | 471/3000 [01:21<04:05, 10.28it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  16%|█▌        | 478/3000 [01:21<02:28, 17.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  16%|█▌        | 481/3000 [01:22<03:44, 11.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  16%|█▌        | 483/3000 [01:22<04:51,  8.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  16%|█▋        | 488/3000 [01:23<04:39,  8.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  16%|█▋        | 493/3000 [01:23<03:23, 12.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  16%|█▋        | 495/3000 [01:23<03:39, 11.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  17%|█▋        | 500/3000 [01:23<02:58, 13.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  17%|█▋        | 504/3000 [01:24<04:13,  9.84it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  17%|█▋        | 508/3000 [01:24<04:47,  8.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  17%|█▋        | 510/3000 [01:25<04:16,  9.71it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  17%|█▋        | 514/3000 [01:25<03:40, 11.28it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  17%|█▋        | 518/3000 [01:25<02:46, 14.92it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  17%|█▋        | 520/3000 [01:25<03:47, 10.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  17%|█▋        | 522/3000 [01:26<04:16,  9.66it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  17%|█▋        | 524/3000 [01:26<04:36,  8.97it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  18%|█▊        | 528/3000 [01:26<04:09,  9.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  18%|█▊        | 530/3000 [01:27<04:16,  9.63it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  18%|█▊        | 534/3000 [01:27<03:53, 10.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  18%|█▊        | 538/3000 [01:27<03:02, 13.52it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  18%|█▊        | 540/3000 [02:00<3:13:55,  4.73s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  18%|█▊        | 544/3000 [02:01<1:22:26,  2.01s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "\u001b[32m2026-01-04 10:46:53.105\u001b[0m | \u001b[31m\u001b[1mERROR   \u001b[0m | \u001b[36mevoagentx.workflow.workflow\u001b[0m:\u001b[36masync_execute\u001b[0m:\u001b[36m104\u001b[0m - \u001b[31m\u001b[1mAn Error occurs when executing the workflow: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 1 second. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  18%|█▊        | 548/3000 [02:02<43:36,  1.07s/it]  "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  18%|█▊        | 552/3000 [02:02<24:18,  1.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  18%|█▊        | 555/3000 [02:03<20:28,  1.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  19%|█▊        | 559/3000 [02:04<12:27,  3.27it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  19%|█▉        | 568/3000 [02:04<05:02,  8.04it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  19%|█▉        | 570/3000 [02:04<04:51,  8.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  19%|█▉        | 573/3000 [02:04<04:21,  9.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  19%|█▉        | 575/3000 [02:05<05:35,  7.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  19%|█▉        | 577/3000 [02:05<05:17,  7.63it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  19%|█▉        | 580/3000 [02:05<05:30,  7.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  20%|█▉        | 585/3000 [02:06<03:41, 10.89it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  20%|█▉        | 589/3000 [02:06<02:48, 14.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  20%|█▉        | 594/3000 [02:06<03:10, 12.66it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  20%|█▉        | 596/3000 [02:07<04:10,  9.61it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  20%|█▉        | 598/3000 [02:07<04:26,  9.02it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  20%|██        | 600/3000 [02:07<04:22,  9.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  20%|██        | 603/3000 [02:08<04:08,  9.67it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  20%|██        | 610/3000 [02:08<02:58, 13.36it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  20%|██        | 612/3000 [02:08<04:25,  9.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  20%|██        | 614/3000 [02:09<04:39,  8.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  21%|██        | 618/3000 [02:09<04:19,  9.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  21%|██        | 620/3000 [02:09<04:06,  9.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  21%|██        | 622/3000 [02:09<04:08,  9.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  21%|██        | 629/3000 [02:10<02:52, 13.72it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  21%|██        | 633/3000 [02:10<03:57,  9.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  21%|██        | 637/3000 [02:11<03:29, 11.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  21%|██▏       | 641/3000 [02:11<03:18, 11.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  22%|██▏       | 645/3000 [02:11<03:24, 11.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  22%|██▏       | 647/3000 [02:12<03:19, 11.82it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  22%|██▏       | 651/3000 [02:12<03:27, 11.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  22%|██▏       | 653/3000 [02:12<03:36, 10.84it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  22%|██▏       | 657/3000 [02:13<03:48, 10.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  22%|██▏       | 661/3000 [02:13<03:47, 10.27it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  22%|██▏       | 663/3000 [02:13<03:46, 10.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  22%|██▏       | 666/3000 [02:14<04:18,  9.02it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  22%|██▏       | 668/3000 [02:14<04:02,  9.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  22%|██▏       | 672/3000 [02:14<03:03, 12.66it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  23%|██▎       | 676/3000 [02:14<02:59, 12.97it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  23%|██▎       | 678/3000 [02:15<03:58,  9.72it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  23%|██▎       | 682/3000 [02:15<03:30, 11.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  23%|██▎       | 684/3000 [02:15<03:07, 12.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  23%|██▎       | 686/3000 [02:15<03:46, 10.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  23%|██▎       | 689/3000 [02:16<03:19, 11.57it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  23%|██▎       | 692/3000 [02:16<04:17,  8.97it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  23%|██▎       | 696/3000 [02:16<04:06,  9.36it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  23%|██▎       | 698/3000 [02:17<03:41, 10.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  23%|██▎       | 702/3000 [02:17<03:21, 11.42it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  23%|██▎       | 704/3000 [02:17<05:22,  7.11it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  24%|██▎       | 706/3000 [02:18<05:26,  7.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  24%|██▎       | 709/3000 [02:18<04:53,  7.82it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  24%|██▍       | 716/3000 [02:18<02:31, 15.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  24%|██▍       | 718/3000 [02:19<06:13,  6.11it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  24%|██▍       | 723/3000 [02:20<04:16,  8.86it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  24%|██▍       | 728/3000 [02:20<03:15, 11.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  24%|██▍       | 730/3000 [02:20<03:14, 11.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  24%|██▍       | 732/3000 [02:20<03:36, 10.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  24%|██▍       | 734/3000 [02:21<05:48,  6.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  25%|██▍       | 738/3000 [02:21<04:04,  9.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  25%|██▍       | 740/3000 [02:21<04:05,  9.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  25%|██▍       | 746/3000 [02:22<02:48, 13.40it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  25%|██▌       | 750/3000 [02:22<02:46, 13.55it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  25%|██▌       | 752/3000 [02:22<03:36, 10.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  25%|██▌       | 754/3000 [02:23<05:39,  6.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  25%|██▌       | 756/3000 [02:23<05:14,  7.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  25%|██▌       | 761/3000 [02:23<03:09, 11.82it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  26%|██▌       | 767/3000 [02:24<02:07, 17.48it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  26%|██▌       | 770/3000 [02:24<02:01, 18.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  26%|██▌       | 775/3000 [02:25<04:36,  8.04it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  26%|██▌       | 777/3000 [02:25<04:21,  8.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  26%|██▌       | 781/3000 [02:25<03:29, 10.61it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  26%|██▌       | 786/3000 [02:26<03:04, 12.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  26%|██▋       | 790/3000 [02:26<02:27, 15.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  26%|██▋       | 792/3000 [02:26<03:10, 11.61it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  26%|██▋       | 794/3000 [02:27<04:45,  7.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  27%|██▋       | 796/3000 [02:27<04:41,  7.82it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  27%|██▋       | 800/3000 [02:27<04:19,  8.47it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  27%|██▋       | 802/3000 [02:27<03:53,  9.40it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  27%|██▋       | 805/3000 [02:28<03:20, 10.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  27%|██▋       | 807/3000 [02:28<04:07,  8.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  27%|██▋       | 810/3000 [03:01<2:13:12,  3.65s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  27%|██▋       | 814/3000 [03:01<1:04:58,  1.78s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  27%|██▋       | 816/3000 [03:02<50:40,  1.39s/it]  "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  27%|██▋       | 820/3000 [03:02<26:12,  1.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  27%|██▋       | 824/3000 [03:02<14:48,  2.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  28%|██▊       | 827/3000 [03:03<13:25,  2.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  28%|██▊       | 831/3000 [03:04<09:11,  3.93it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  28%|██▊       | 833/3000 [03:04<07:27,  4.84it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  28%|██▊       | 835/3000 [03:04<06:44,  5.35it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  28%|██▊       | 839/3000 [03:04<04:26,  8.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  28%|██▊       | 841/3000 [03:05<05:01,  7.17it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  28%|██▊       | 843/3000 [03:05<04:05,  8.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  28%|██▊       | 845/3000 [03:05<04:57,  7.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  28%|██▊       | 847/3000 [03:06<04:48,  7.46it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  28%|██▊       | 853/3000 [03:06<02:56, 12.14it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  28%|██▊       | 855/3000 [03:06<03:41,  9.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  29%|██▊       | 859/3000 [03:06<03:18, 10.80it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  29%|██▊       | 861/3000 [03:07<03:05, 11.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  29%|██▉       | 863/3000 [03:07<03:50,  9.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  29%|██▉       | 867/3000 [03:07<03:29, 10.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  29%|██▉       | 871/3000 [03:07<02:34, 13.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  29%|██▉       | 875/3000 [03:08<02:43, 13.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  29%|██▉       | 877/3000 [03:08<02:41, 13.14it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  29%|██▉       | 879/3000 [03:08<04:04,  8.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  29%|██▉       | 881/3000 [03:09<04:14,  8.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  30%|██▉       | 889/3000 [03:09<02:57, 11.86it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  30%|██▉       | 891/3000 [03:09<03:15, 10.80it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  30%|██▉       | 893/3000 [03:10<03:10, 11.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  30%|██▉       | 897/3000 [03:10<03:11, 11.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  30%|██▉       | 899/3000 [03:10<02:49, 12.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  30%|███       | 901/3000 [03:10<03:16, 10.67it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  30%|███       | 903/3000 [03:11<03:12, 10.87it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  30%|███       | 907/3000 [03:11<03:24, 10.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  30%|███       | 909/3000 [03:11<03:50,  9.06it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  30%|███       | 913/3000 [03:12<03:07, 11.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  30%|███       | 915/3000 [03:12<03:45,  9.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  31%|███       | 917/3000 [03:12<03:43,  9.32it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  31%|███       | 921/3000 [03:12<03:28,  9.97it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  31%|███       | 924/3000 [03:13<03:13, 10.74it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  31%|███       | 928/3000 [03:13<03:05, 11.16it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  31%|███       | 932/3000 [03:13<02:55, 11.80it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  31%|███       | 934/3000 [03:14<03:13, 10.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  31%|███       | 936/3000 [03:14<03:00, 11.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  31%|███▏      | 938/3000 [03:14<03:15, 10.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  31%|███▏      | 940/3000 [03:14<03:28,  9.86it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  31%|███▏      | 943/3000 [03:15<04:07,  8.32it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  32%|███▏      | 945/3000 [03:15<04:08,  8.28it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  32%|███▏      | 948/3000 [03:15<02:56, 11.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  32%|███▏      | 950/3000 [03:15<02:47, 12.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  32%|███▏      | 956/3000 [03:16<02:08, 15.88it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  32%|███▏      | 958/3000 [03:16<03:18, 10.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  32%|███▏      | 960/3000 [03:16<03:39,  9.28it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  32%|███▏      | 962/3000 [03:17<04:08,  8.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  32%|███▏      | 964/3000 [03:17<03:43,  9.11it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  32%|███▏      | 966/3000 [03:17<03:53,  8.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  32%|███▏      | 970/3000 [03:17<03:01, 11.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  32%|███▎      | 975/3000 [03:18<03:39,  9.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  33%|███▎      | 977/3000 [03:18<03:53,  8.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  33%|███▎      | 980/3000 [03:19<03:57,  8.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  33%|███▎      | 987/3000 [03:19<02:09, 15.57it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  33%|███▎      | 991/3000 [03:20<04:37,  7.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  33%|███▎      | 996/3000 [03:20<03:05, 10.83it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  33%|███▎      | 1000/3000 [03:20<02:18, 14.48it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  34%|███▎      | 1005/3000 [03:22<04:53,  6.80it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  34%|███▎      | 1012/3000 [03:22<02:39, 12.49it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  34%|███▍      | 1017/3000 [03:22<01:51, 17.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  34%|███▍      | 1021/3000 [03:23<04:09,  7.92it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  34%|███▍      | 1027/3000 [03:23<03:11, 10.28it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  34%|███▍      | 1029/3000 [03:24<03:22,  9.74it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  34%|███▍      | 1035/3000 [03:24<02:13, 14.71it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  35%|███▍      | 1038/3000 [03:24<02:18, 14.14it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  35%|███▍      | 1041/3000 [03:25<03:23,  9.63it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  35%|███▍      | 1043/3000 [03:25<03:34,  9.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  35%|███▍      | 1047/3000 [03:25<03:12, 10.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  35%|███▌      | 1052/3000 [03:26<02:48, 11.58it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  35%|███▌      | 1058/3000 [03:26<01:58, 16.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  35%|███▌      | 1060/3000 [03:26<02:58, 10.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  35%|███▌      | 1064/3000 [03:27<03:16,  9.84it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d9a9350>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056ee7cb0>, 1554874.377324109)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d9ab310>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d7fd090>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303fce7540>, 1554875.478708333)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d9d8890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d82d3d0>\n",
      "Evaluating workflow:  36%|███▌      | 1066/3000 [03:28<05:42,  5.64it/s]Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056e048a0>, 1554875.769401724)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302dafd350>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d9376d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056e06270>, 1554875.777744561)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d84f190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d8303d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056c50c20>, 1554875.579609276)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d830750>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302da02310>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056ee52b0>, 1554874.536873762)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d9c4390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d7f2150>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530ffbdd080>, 1554910.28502888)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d8a3450>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d6ffe50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303f1ed160>, 1554910.010692065)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d830910>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d75de10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530ff9deac0>, 1554911.850626389)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d75de90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d752350>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d76c790>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d715450>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530ff9dcbb0>, 1554911.706274562)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302db8ae50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530ff9dc130>, 1554911.760791014)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d752250>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530ff9dc8a0>, 1554911.553902724)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d76cc50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d614c10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048c0d4e0>, 1554913.160534983)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d617d50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d629c10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056e04360>, 1554913.38461561)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d7512d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d611310>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056ee73f0>, 1554913.002339265)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d7d3b10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d7502d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303cf45010>, 1554913.165837456)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d759d90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d65d290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303cf45fd0>, 1554913.815793472)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d61d190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d7cd310>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d63bf50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530ff9de430>, 1554913.518636097)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d7cd890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056e04520>, 1554913.551392763)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d63be50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d6e2890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530ff9dc2f0>, 1554913.532907394)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d6e2a10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d566ad0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302d5266d0>, 1554915.238962252)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d566710>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d630910>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048c0cd00>, 1554913.66386245)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d760590>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d591990>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048c0f9a0>, 1554916.010569915)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d5925d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d5e85d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302d5b6f90>, 1554916.895806908)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d5e8f90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d5668d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056e05a20>, 1554917.894177402)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d52f0d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d505350>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302d526200>, 1554917.0713883)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d42f110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d588510>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d58ad90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d59fed0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d5ab1d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048c0d5c0>, 1554915.573685345)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d54ad50>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302d527310>, 1554915.5559199)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d588d10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302d527b60>, 1554915.205391122)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d59fdd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302d526f20>, 1554915.541270583)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d5aafd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d460910>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302d4491d0>, 1554918.214468676)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d4607d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d4769d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302d44a580>, 1554918.291350973)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d476910>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d4ac250>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302d44b8c0>, 1554917.446195996)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d4ac510>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d4716d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302d524c20>, 1554917.973732407)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d5eadd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d4566d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d46d350>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302d448fa0>, 1554916.918930743)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d456950>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302d525630>, 1554918.046153966)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d46d210>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d375b10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302d4c8e50>, 1554919.60246546)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d377a90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d37bdd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302d5b6970>, 1554920.092982942)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d37bf10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d348390>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302d44b4d0>, 1554918.769340464)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d349610>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d3366d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302d34c910>, 1554921.396350054)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d3d3a10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d369cd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302d3db9a0>, 1554920.928079971)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d345590>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d3e4dd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302cc0f350>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302d525b70>, 1554920.169298903)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d3e5510>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302d3d95c0>, 1554921.474709395)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302cc0f210>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302cc9ac10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302cc71b00>, 1554921.913016283)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302cc9ab10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302cc7e610>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302cc72270>, 1554921.656098624)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302cc7e750>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302ccd59d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302cafcc20>, 1554922.591535093)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302cc58810>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302cbded10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302cafcfa0>, 1554924.129566654)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d5d2f50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302ca3f050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302ca38980>, 1554924.743508082)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302ca3ee90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302ca823d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302ca39f60>, 1554924.918222973)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302ca82750>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302ca66d10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302ca3ab30>, 1554924.954619273)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302ca66390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302c9775d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302cade7b0>, 1554927.266480067)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302c977490>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302c950510>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302c98c4d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302cadfbd0>, 1554927.251897805)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302c950b90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302cc73700>, 1554927.084159415)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302c98c810>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302c960d50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302c994750>, 1554927.137973787)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302c9609d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302c962f10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302c9b3190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302d4c8c90>, 1554927.250453974)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302c963e10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302c995080>, 1554927.117596784)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302c9b3110>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302c9abb10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302c9de4d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302c995390>, 1554927.255015501)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302c9c4350>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302c995da0>, 1554927.690535911)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302c9de9d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302c7fd2d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302c996820>, 1554927.489179565)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302c7fdad0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302cb640d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302c995be0>, 1554927.531961228)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302c9c4750>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302c80ded0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302c828a90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302c996120>, 1554928.805394023)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302c80dad0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302c93d050>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302c956c50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302c83c3d0>, 1554927.581418114)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302c828950>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302cade900>, 1554926.166293688)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302c93f4d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302cadf380>, 1554926.845056337)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302c956a50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302c83a3d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d9bcd10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302da98190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d9cba90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d9e0c50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d992a90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d821f50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d83ce50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d804b90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d870b10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d82ca90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302da65f10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d858dd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d8a0ed0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d8ac8d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d8ba610>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d9abed0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d8ccd90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d8d8dd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d70cbd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d72b4d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d7238d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d740b10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d889050>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d7215d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d7f7190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d7504d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d76ecd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d5f8750>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d5f3910>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d672a10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d684ad0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d650b50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d68b6d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d6a3550>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d6b4d10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d6c6150>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d65e910>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d6d6510>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d663ad0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d6e0550>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d658450>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d608950>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d522090>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d52e210>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d684190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d650190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d695650>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d56e850>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d612990>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d516850>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d5566d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d565e10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d5804d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d5c5f50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d5d7e10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d565110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d3f9550>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d408750>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d43ef10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d44d950>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d474890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d48dc50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d49a210>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d4a6110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d4e50d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d408550>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d2f6450>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d304a10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d4bcc90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d685c10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d335210>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d455f10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d4a7250>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d363c50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d348dd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d383810>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d391c90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d3b0ed0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d3c0090>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d324e50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d3ace50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d39d990>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302cc18610>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d304990>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302cc266d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302cc28250>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d335050>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302cc3be90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302cc464d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302cc58c10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302cc78dd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302cc89790>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d3b1b10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302cca4650>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302cccdb50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d35c390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302cccd310>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302caf7e10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302cb02690>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302cb1a0d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302cb29f90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302ccdf010>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302cb48cd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302ccbba10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302ccc5750>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302cb50ed0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302cb5b490>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302cb36290>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302cb90d90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302cba8390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302cbb9390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302cbc8650>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302cc8f210>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302ca17290>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302ca34f90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302cb819d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302cb7cc50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302cb926d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302cbdd790>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302c9f69d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302ca40f90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302ca5f410>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302ca7bf90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302ca8e690>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302cab43d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302cac9650>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302c8f1990>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302cab9c90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302c8fa990>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302c90f410>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302cb67110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302cab1190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302ca21250>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302c93a550>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302cadd710>, 1554926.837319137)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302c90f2d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302cade120>, 1554926.828558529)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302c9f5a10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302ca3a580>, 1554926.830046825)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302ca8e490>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302caff700>, 1554928.387549023)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302c839e50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056ee6430>, 1554874.242979491)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d9bd210>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056ee4a60>, 1554874.159194592)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302da98250>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048f273f0>, 1554874.583242957)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d9cb950>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056ee4c90>, 1554874.716472055)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d9e09d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056ee4bb0>, 1554875.411937131)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302da27c10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303f1ec4b0>, 1554875.758561583)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d8223d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056e07af0>, 1554875.606780693)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d83d3d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056e05630>, 1554874.900259347)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d804ed0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056e047c0>, 1554875.451352423)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d821050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056e04440>, 1554875.791796427)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d827d50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056ee4d70>, 1554875.267612925)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d7f2dd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056c52a50>, 1554910.355912236)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d858bd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530ffbdfaf0>, 1554911.689748726)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d8a1590>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530ffbdc7c0>, 1554910.292370865)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d8acdd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056e06820>, 1554876.008065115)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d8baad0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303fce7a10>, 1554911.472397396)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d9aaf50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530ffbdc210>, 1554910.454512232)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d8ccb90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048f26eb0>, 1554911.705208411)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d8d8c90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056e06190>, 1554910.162073179)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d70ca50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530ff9dc670>, 1554909.902650196)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d72b390>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530ff9dce50>, 1554911.569097097)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d823c90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530ff9dd320>, 1554911.587805235)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d7409d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530ffbddc50>, 1554911.549960154)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d888cd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303fce6580>, 1554911.644513266)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d721490>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530ff9de040>, 1554911.584541535)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d7f7e50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530ff9de510>, 1554909.866901285)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d808890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15310389e3c0>, 1554911.445697928)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d76ec90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048c0cde0>, 1554911.822059762)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d5f84d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15310389db70>, 1554911.925026612)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d5f3890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048c0d780>, 1554913.13238246)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d672810>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056e04f30>, 1554913.538118048)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d7b4b10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530ff9dd940>, 1554913.462701695)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d650ed0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303cf45390>, 1554913.008006476)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d68b590>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303cf44d70>, 1554913.522323797)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d6a3410>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048f270e0>, 1554913.195689506)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d6b4bd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303cf44670>, 1554914.07741722)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d6c6810>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303cf44590>, 1554913.239761943)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d65fb90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530ff9dd8d0>, 1554913.656348499)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d6d63d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303cf45940>, 1554913.849263254)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d6638d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056e05e10>, 1554913.568521831)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d6e0c10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048c0d7f0>, 1554915.02501695)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d658150>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15310389c1a0>, 1554914.608076808)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d505310>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302d524750>, 1554914.765508629)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d521210>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302d524c90>, 1554915.332738546)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d52e010>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302d524ec0>, 1554914.980310855)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d685dd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302d525860>, 1554915.869515437)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d684710>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302d5264a0>, 1554916.044442693)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d651e50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303cf45e10>, 1554915.28653362)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d56e310>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303cf45400>, 1554914.794669091)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d612290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303cf46270>, 1554914.979156669)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d516650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15310389d940>, 1554915.213597858)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d557c90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303cf47d20>, 1554915.81927213)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d5108d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302d526c10>, 1554915.389105669)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d750610>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302d5b4980>, 1554915.665470403)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d5c5e10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302d5b5630>, 1554916.483068375)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d5d7d10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302d5b5fd0>, 1554916.575079458)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d61ce10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302d5b6200>, 1554916.657753428)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d3f9410>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302d5b7150>, 1554916.692389957)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d608490>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302d448590>, 1554916.987412022)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d43ed10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303cf46900>, 1554917.414509404)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d44d7d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302d44a350>, 1554917.220780737)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d4746d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302d44a7b0>, 1554917.501185298)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d48db10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302d449a20>, 1554917.878604445)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d49a0d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302d44b620>, 1554918.522322391)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d4a5fd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302d4c8fa0>, 1554918.27139829)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d4e5790>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302d4c8520>, 1554918.932449502)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d408ad0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302d5b57f0>, 1554918.688044779)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d2f6250>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302d4c9ef0>, 1554918.839102629)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d304890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048c0e190>, 1554917.756265508)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d4bca90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302d4492b0>, 1554918.990271992)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d632510>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302d4cb690>, 1554919.025150629)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d3355d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302d448d00>, 1554919.021325804)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d4564d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302d34ca60>, 1554919.117265172)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d3554d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302d34cc90>, 1554919.052427862)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d363b10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302d34d5c0>, 1554919.132785821)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d348fd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302d34dd30>, 1554920.350854527)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d383610>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302d34d2b0>, 1554919.926795331)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d392250>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302d44b070>, 1554919.939786241)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d3b1590>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302d34e580>, 1554920.433773398)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d3c38d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302d4caf90>, 1554918.846728708)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d325390>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302d4c8210>, 1554919.830840619)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d3ad890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302d34fd90>, 1554919.709845589)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d39ffd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302d3d9c50>, 1554920.629810951)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302cc17f50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302d3d9e10>, 1554920.671071213)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d304a90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302d3da4a0>, 1554920.603727714)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302cc25e50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302d3da970>, 1554920.888013109)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d325bd0>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302d3daf20>, 1554921.679491896)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302cbfc510>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302d3d8f30>, 1554920.657748572)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302cc388d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302d3db1c0>, 1554920.73531745)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302cc462d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302d3d81a0>, 1554920.802586043)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302cc58a10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302cc70910>, 1554921.865201013)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302cc78c90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302cc71080>, 1554921.323529992)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302cc89650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302cc716a0>, 1554921.595263817)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d3b1bd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302cc71cc0>, 1554921.781961377)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302cca4d50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302cc737e0>, 1554922.252257839)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302cccd950>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302cc71400>, 1554922.016965099)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d35c990>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302cc72c80>, 1554922.637413858)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302cc470d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302cafc750>, 1554922.874982746)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302caf7cd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302cafc980>, 1554922.577858359)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302cb02490>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302cafd2b0>, 1554922.438112359)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302cb19ed0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302cafda90>, 1554923.102675419)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302cc45190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302cafdc50>, 1554922.757344014)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302ccdf090>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302d34e040>, 1554923.523116895)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302cc40d10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302cc72820>, 1554922.081216441)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302ccb8dd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302d448440>, 1554922.597053229)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302cb1a9d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302cafee40>, 1554923.254945696)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302cb50d10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302caff460>, 1554923.440608176)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302cb5b3d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302caffbd0>, 1554923.357884661)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302cb367d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302cc72510>, 1554923.398960002)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302cb90c50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302cb8cd70>, 1554923.685955344)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302cba8ad0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302cb8d400>, 1554923.922085066)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302cbb9890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302cb8d940>, 1554924.064055301)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302cbc8d90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302cb8ef20>, 1554924.880924609)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302c9f5890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302cb8f930>, 1554924.289551845)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302ca17850>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302ca38130>, 1554924.537570515)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302ca35910>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302cb8c520>, 1554923.495230354)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302cb817d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302cc70e50>, 1554923.96546612)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302cb7ccd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302cc71010>, 1554924.155613444)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302cb1ad10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302cb8f7e0>, 1554924.27525142)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302ccc8c90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302cb8faf0>, 1554924.433835137)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302c9f5fd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302cafcc90>, 1554924.75360023)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302ca21410>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302ca386e0>, 1554924.850069322)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302ca5f250>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302ca39d30>, 1554925.289926394)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302ca7be50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302ca3b310>, 1554925.294546393)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302cab4ad0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302ca3be00>, 1554925.765713604)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302cac9450>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302cadc590>, 1554926.836301036)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302c8f1850>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302d4c9da0>, 1554925.611421892)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302cb7f110>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302cb8d7f0>, 1554925.916169818)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302c8fa290>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302cadd8d0>, 1554925.489764037)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302cb67dd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302ca3b0e0>, 1554926.827062307)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302cab0e10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302cade2e0>, 1554926.10732258)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302c93a350>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d893810>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056e064a0>, 1554875.682226579)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d893fd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302c9a79d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302ca3af90>, 1554929.278551465)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302c8e6c50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302c7f4990>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303cf45710>, 1554929.277003286)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302c7f48d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302c9571d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302c8190d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302c9b4090>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302c997700>, 1554929.150280636)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302c878a90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15310389c750>, 1554929.401623905)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302c805550>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302c7327d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fe39d0f0>, 1554929.230886463)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302c732550>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302cb42a50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302cc27010>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fe39cec0>, 1554929.323814497)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302cb534d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fe39e6d0>, 1554929.683048407)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302cc273d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302c8384d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302c9955c0>, 1554929.471100533)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302c839d10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d427b90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d5e60d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd7589f0>, 1554930.091328232)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d3f8510>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd759400>, 1554930.267517785)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d5e52d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d5124d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d6c2310>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd759a20>, 1554930.450461264)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d512410>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d64ca50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531079bb7e0>, 1554930.393191867)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d64eb10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d625350>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d61ce90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd346820>, 1554930.494749041)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d624f90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd3441a0>, 1554930.632622367)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d61ca50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d7e4c50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302c89e910>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302c8baed0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302c8c5b10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302c6f7b90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302c70cf50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302c6f4dd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302ca3e6d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15310389e580>, 1554929.370944361)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302c9b4dd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302c98c5d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303ef03460>, 1554928.888095652)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302c8b3690>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531079ba4a0>, 1554930.928189396)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302cab1a10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15322cce8de0>, 1554930.418030699)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d7e4d90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531044ea430>, 1554928.434922093)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302c89ef10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531038d5550>, 1554928.722664706)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302c8c6090>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057fdd6a0>, 1554928.569806342)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302c8d2dd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303d4504b0>, 1554928.774769858)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302c8d1b50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531026a7c40>, 1554928.728833411)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302c703d90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303cf45320>, 1554928.746377058)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302cba2150>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048c0dbe0>, 1554929.090476612)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302c8e7bd0>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15305506d150>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303cf47850>, 1554929.128825446)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302cafaed0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302c887950>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fca3f460>, 1554930.869721147)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302c887a90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d7e7890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15310389eeb0>, 1554930.755720771)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d87c710>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d859950>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fca3e3c0>, 1554930.834788325)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d859910>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530542f66d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d983a90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302ca3ba10>, 1554931.069628614)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302ccbb850>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fcee2e40>, 1554931.451809862)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d973950>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302c8f1690>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d955550>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fcee0e50>, 1554930.892334711)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302c8f2b50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fcee1320>, 1554931.184280188)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d955890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302c8b04d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d937c50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fca3cfa0>, 1554931.876292697)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d983610>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d958650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057fdd160>, 1554930.935012291)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d7ab310>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d979a10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057fdd320>, 1554931.589093652)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d95bd10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d554450>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fccb0fa0>, 1554931.960872416)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d5549d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302dae0250>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd759390>, 1554931.968067597)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d7e5890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302daa8950>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd5047c0>, 1554932.380394757)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302da9f1d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d831550>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fe39f460>, 1554930.966460638)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d935990>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd505f60>, 1554932.33797612)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302daa8750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fca3e4a0>, 1554930.697474079)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302c8d1810>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302daa5190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153102750de0>, 1554932.974204075)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302da984d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302da7ccd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd75baf0>, 1554932.44837497)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302da7d210>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302da72050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd1da2e0>, 1554932.658530866)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302da71bd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302da1d1d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fca3f620>, 1554933.167630626)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302da1fb90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302dbded50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd1db150>, 1554932.491566239)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302dbdcf10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d9f7510>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302dbc9ed0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fca3e970>, 1554932.457254349)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d9f7890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fca3cd00>, 1554933.009789609)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302dbc9f90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302dbcbd90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302da11250>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd75ad60>, 1554932.90473259)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302da06cd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056ddb9a0>, 1554932.76847861)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302da116d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153049b64d90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302db96810>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302cc73310>, 1554932.565545453)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d95b1d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302db87210>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd6b7620>, 1554932.690662666)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302db861d0>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302dbaefd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302c996660>, 1554933.053470892)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302db77f90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530499df8d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302db40cd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fcee1c50>, 1554933.302326508)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d95b750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b659160>, 1554933.582190836)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302db41350>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302db29ed0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fcee27b0>, 1554932.90790708)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302db954d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b65a190>, 1554933.889196828)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302db28a10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302dad0390>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b65a900>, 1554933.754289712)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302dad3150>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302db4c950>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b659cc0>, 1554933.691896124)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302db4c510>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530341d9fd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b659f60>, 1554934.202629354)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530341daf90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302da66d50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd1d92b0>, 1554934.324023782)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302dae2e50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530341c3d90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b658de0>, 1554934.653477783)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530341c1790>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302db0d4d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153034116b90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd1da430>, 1554934.32817113)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530498c6050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd221d30>, 1554934.693087683)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153034116690>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530341e7490>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153034153a50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15322d85cbb0>, 1554934.387285221)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530341e4c90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531079ba900>, 1554934.691976513)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153034151bd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530341d1150>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303413dc10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd222e40>, 1554934.959573107)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530341d0b90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153034176050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd222890>, 1554934.793937274)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153034176010>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302da87c10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153034182950>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd222120>, 1554934.856216668)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302da87310>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057fdd710>, 1554934.852171428)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153034182a50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303410a6d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056dda900>, 1554934.475418416)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303413dcd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd223380>, 1554935.063518899)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530341bab50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302dbaed90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153034cca690>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd222d60>, 1554934.799812685)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302dbaec50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153034cfab90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd220590>, 1554935.568083225)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153034cfb190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153034ce6110>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153105245470>, 1554935.293517387)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153034ce5610>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153034123550>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153034ce02d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531052459b0>, 1554935.833638094)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d90c350>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd223f50>, 1554935.889546858)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153034ce1550>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d90cbd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153105246b30>, 1554935.704789039)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d90cb50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153034cafd10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b8563c0>, 1554936.207166221)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530341f7190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530341911d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b855390>, 1554936.061547736)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153034c97850>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153034cad110>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b65a4a0>, 1554936.793298859)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302db95e10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530341da510>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153105246f90>, 1554936.110308459)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530341db010>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153034c5e890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b856270>, 1554936.318233381)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153034c5c590>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153034c2ee90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153034c0efd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd2207c0>, 1554936.452837809)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153034c2ec10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153034cc1010>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b65ae40>, 1554935.363663725)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153034ccbe10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b8570e0>, 1554970.391788645)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153034cc9410>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b854360>, 1554936.429113365)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153034cc0d90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153034c68710>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b854830>, 1554936.977013473)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153034c7ef10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153034c20d50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153034c1c3d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd5db070>, 1554936.760714544)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153034c2af50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd5da510>, 1554936.87318979)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153034c1ee90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153035717cd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530357684d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd2223c0>, 1554936.914426819)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153035716b90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530358a6490>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530ffbde3c0>, 1554970.333068756)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530358a5450>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530358c32d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048f26820>, 1554970.56694933)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530358c2990>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303581ac10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fca3c0c0>, 1554971.619074982)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303581ba10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530341d9f90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530357a1250>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd223770>, 1554970.474469544)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530341d8510>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048f259b0>, 1554971.877088586)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530357a3290>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530355b6450>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048f27310>, 1554972.151723865)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530355b4a50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153035562c10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048f26a50>, 1554970.449123729)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153035562dd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530355d5ed0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048f24c20>, 1554970.216718409)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530355d5d10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153035545890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153034c3d110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153034c21210>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd220d00>, 1554971.824076777)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530357690d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b855400>, 1554936.56423961)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153034c3ccd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd221400>, 1554971.796747838)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153035546090>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd222190>, 1554936.634614442)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153034c23350>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153035716e90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056e066d0>, 1554971.861837172)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530354b74d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303525b990>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303fce7700>, 1554973.549974379)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303525b810>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153034c55fd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303fce6120>, 1554973.601574938)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153034c57950>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153035560b50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303f2566d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530354ee610>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530353ea1d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530352c9ad0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530356f1790>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153034ff66d0>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303508b950>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303fce6890>, 1554971.758745071)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153034cdae50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531038d7230>, 1554972.114257737)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153035088610>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303fbb7750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b856eb0>, 1554973.614073217)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153035562d90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531038d6740>, 1554973.308497245)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303f255950>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056ee43d0>, 1554972.190981074)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153035327710>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd5041a0>, 1554973.747066677)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530353ead10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303fce6820>, 1554973.950907603)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530352cbdd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd221550>, 1554971.128900111)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153034ff4450>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303fce63c0>, 1554973.074955881)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530549b0610>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fdc26e90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd8d4750>, 1554973.290534644)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fdc24710>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153034d64f10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd5da7b0>, 1554973.419484184)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153034d67910>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531051aee10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153103d06e40>, 1554973.579460275)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531051af0d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303fb1b690>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304be491d0>, 1554973.765761046)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304b58e4d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153034fa02d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153034141ad0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304be48520>, 1554973.973847608)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304936b410>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304be49630>, 1554973.606705575)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304b6c0b10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fcc0e310>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304be48e50>, 1554973.575638447)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303f422b90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153055102710>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304be4aba0>, 1554974.02114637)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153055103550>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304be257d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530493b4e10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304be49860>, 1554974.183831222)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304beff910>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048f26900>, 1554974.363492753)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530494dd190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153055440310>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304be4a580>, 1554974.681608078)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153034c38690>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303fa484d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153048851c90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530359724d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304be4bb60>, 1554973.441402382)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303faa45d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304be48bb0>, 1554973.38826792)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153035c84c50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531038d40c0>, 1554973.475368275)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530355d7b50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153056fc9850>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd8d54e0>, 1554974.753820637)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303fb97410>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530491aca50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530355d6e10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531038d4600>, 1554974.897710018)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153048855c10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531038d7310>, 1554975.347383831)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530355d7bd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530551b8bd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530428e8a90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153103d04210>, 1554975.353140864)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530551ba2d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056c52740>, 1554975.86191536)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530428e8d50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530556e2a90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153103c037e0>, 1554975.616031299)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530556e1990>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15305531c910>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153034c1ba50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531038d6430>, 1554976.099964743)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15305531c790>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304be48ad0>, 1554975.074912383)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153055b21fd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303f2dc310>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15305559c290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531038d5390>, 1554975.827765154)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303f2dd0d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd8d5b70>, 1554975.464226652)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153055a4d890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304b8070d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530556d3810>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153103c00830>, 1554975.439851492)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153034d9c190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153100fc07d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153055b9fe10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153103c03460>, 1554975.646712219)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530354b61d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153103c023c0>, 1554975.602887739)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530554e0950>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304beffdd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153055c26b10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153103c02f90>, 1554975.638056788)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304b3f0990>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056e05940>, 1554975.783586361)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531044a84d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153055b31490>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153103c016a0>, 1554975.23184926)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530556d1510>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153103c00440>, 1554975.643536408)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153055b31ed0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fdc25dd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153055e49990>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153103c01fd0>, 1554976.601728197)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153055cd05d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304be492b0>, 1554976.152342252)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153055e4b510>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153055fdbcd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531044e9f60>, 1554976.544712422)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153055e9f490>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303fc26390>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153103c00ad0>, 1554976.242879581)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303fc25750>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153055d06690>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531044e9390>, 1554976.763496081)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153055d06bd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303e5afd90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531038d5860>, 1554976.861591113)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303e5af950>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303e47e890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531044eb620>, 1554977.352216956)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303e47cd10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303e6599d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531044e96a0>, 1554976.913724912)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303e65b550>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153055b00a90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531044e8f30>, 1554977.09639375)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304be54690>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303e3339d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303fbf6750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531044ea270>, 1554977.071911164)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303e3322d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531044e9860>, 1554977.409215065)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303fbf4ad0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303d022890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531044ea2e0>, 1554977.1184246)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303d0766d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303fc18850>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303d05e150>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531044e8830>, 1554977.738397992)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303fc19250>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304be48b40>, 1554977.405120025)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303d05e490>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303e598350>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153104f16120>, 1554977.581020033)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303d07e910>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153104f16820>, 1554977.571979952)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303e598c90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303e7c9d10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304be48c90>, 1554977.435803264)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303e7c8390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153055e4b050>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303d0ac450>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153104f16eb0>, 1554978.20924468)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153055e482d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153104f174d0>, 1554978.234561231)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303d0af650>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303e3b6fd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153104f16ba0>, 1554977.976888105)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153048c49dd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303d1b6590>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153104f17770>, 1554978.596933407)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303e5fedd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303d181590>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303e6b09d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153104f14050>, 1554978.480283686)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303d181050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531043d3a10>, 1554979.301532588)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303e6b3350>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303cefe9d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531043d2e40>, 1554979.071639485)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303cefd390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303cf2be50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531043d12b0>, 1554978.796030486)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303cf2ac50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303d09a290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531038d60b0>, 1554979.089172381)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303e37a150>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153055b46150>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303cef0110>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531043d1390>, 1554978.989119441)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153055b45f50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531044eba10>, 1554978.892117003)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303cef36d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303d0847d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531043d0830>, 1554978.727869386)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303d0878d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303d149790>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303cf82a10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531044e9940>, 1554979.286209953)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15305564f190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153103c03ee0>, 1554977.631344774)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303d1d5890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303cf5a990>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531043d10f0>, 1554980.267008025)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303cf5bf50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303cf85c50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303d2e79d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530488ba270>, 1554979.641561453)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303cf85d10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531044e99b0>, 1554979.375900033)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303d2e5550>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303d1698d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530488bad60>, 1554979.876152066)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303e50bbd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303fc24ed0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd8d64a0>, 1554979.350580779)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303d2b5f90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303d2cd710>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303d250310>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530488b8f30>, 1554979.498253244)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303d2ce810>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530488bb850>, 1554980.255851939)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303d252550>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303d20a150>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530488b9630>, 1554980.146744549)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303d208890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303d208950>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530488b9c50>, 1554980.189642421)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303d22b690>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303e57a250>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530488b8280>, 1554980.544019846)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153055d06a90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303d370150>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bf1fc40>, 1554980.71934254)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303d3704d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303d247e50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bf1e040>, 1554980.293661934)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303d244150>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303d333790>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303d303150>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303d1c1a10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303fc10fd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153104f16e40>, 1554980.260378523)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303d3454d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531043d3af0>, 1554980.887744974)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303d302a50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531043d00c0>, 1554979.331616706)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303d1c28d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531043d38c0>, 1554979.554300122)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303fc13490>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303d4ced90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530488bb0e0>, 1554980.875732217)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303d4ce310>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303d034f10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bf1fe00>, 1554981.141303591)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303d0348d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303d499650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bf1fe70>, 1554980.867724525)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303d49a850>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303d48bbd0>\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303d447010>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bf1e6d0>, 1554980.76739324)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303d48b990>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bf1d1d0>, 1554981.235486925)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303d447450>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303e48d490>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303d43e150>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bf1d2b0>, 1554981.056529743)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303d222a50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bf1def0>, 1554981.29621775)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303d43ef10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303d07f250>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303d5aab90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bf1d0f0>, 1554981.718946921)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303d07cfd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304be4a660>, 1554981.393811399)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303d5a8b50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303d5bff90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153105036cf0>, 1554981.451241701)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303d5bde50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303d0fc390>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153105037700>, 1554981.759573077)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303d0fd550>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303d548650>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303d55a5d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531050374d0>, 1554982.208725692)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303d559410>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303d247f50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303d52d490>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531044e8c20>, 1554982.070527334)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303d247d10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153105034910>, 1554981.963641207)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303d208250>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303d58cc10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303d68fbd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153104f15390>, 1554981.571078713)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303d549590>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153105035e80>, 1554982.356503615)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303d58e310>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531043d1010>, 1554981.761553854)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303d68d890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303d6465d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303d2ec510>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303d5f5690>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531044ea200>, 1554982.666310691)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303d302ad0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153105035fd0>, 1554982.387661972)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303d60e6d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303d7caed0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048975550>, 1554982.956719036)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303d7c9350>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303d2e44d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303d79cd90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153105037d90>, 1554982.801552043)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303d2e6290>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303d4fb410>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303d764090>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bf1cec0>, 1554983.000427754)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303d4fb610>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048977a80>, 1554982.992245349)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303d766f90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303d5f7f10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303d6fe110>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bf1eb30>, 1554983.57317219)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303d5bd890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048974ad0>, 1554983.393957663)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303d6fe310>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303d8d6e50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048977e70>, 1554983.59331077)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303d8d6dd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303d89d6d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048976c80>, 1554983.289746622)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303d89dfd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303d876b10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531050348a0>, 1554983.476872919)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303d877f10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303d8e7410>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b558130>, 1554983.663054127)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303d7fa1d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530488bbbd0>, 1554982.39470795)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303d43ce90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048975400>, 1554982.763077296)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303d79c250>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b55be00>, 1554984.105779219)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303d8e7350>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303d87da90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b559da0>, 1554983.650769875)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303d87ce90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303d253d90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b558b40>, 1554984.265871352)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303d253650>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303d9a2b10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303d966bd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b558f30>, 1554983.785518981)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303d9a0c90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bf1ca60>, 1554984.004891559)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303d966850>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303d97eb10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b55a270>, 1554983.921760945)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304ba65750>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303d95f5d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303d919e10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b558600>, 1554984.44351868)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303d91a210>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303da808d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048974910>, 1554984.811359464)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303da81310>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303dab1ed0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303da9f410>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048b27af0>, 1554984.623768539)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303daa84d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303d996ad0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303da71f50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048976120>, 1554984.878127232)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303d908c50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048b279a0>, 1554984.693958638)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303da71b90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303dad7050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048b27c40>, 1554984.583868688)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303dad6950>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303d8d6c90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303dba3b50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530488b8bb0>, 1554984.04438996)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303d95fd50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048976660>, 1554984.766020858)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303dab0150>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531043d01a0>, 1554985.361619916)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303d9eff50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b55a820>, 1554985.217263076)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303dba19d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303db99550>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303db63d50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530489759b0>, 1554985.157234011)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303db99990>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048b26ac0>, 1554985.407801722)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303db60d90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303d9efed0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303db1df90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048b25320>, 1554985.387226546)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303d9ec310>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048b277e0>, 1554985.180822113)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303db1ed90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303d954650>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303dc997d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b55b620>, 1554986.243704904)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303d9560d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b5583d0>, 1554986.642852579)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303dc9a110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303dc7ac90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303d1437e0>, 1554985.832250552)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303dc7b250>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303dcbfe50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303dc38710>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303d140050>, 1554986.099164931)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303dcbc590>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303d143620>, 1554986.057062678)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303dc38550>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303dc1bad0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303d141b70>, 1554985.994682936)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303dc18690>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303dbf74d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b559a20>, 1554986.290920802)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303dbf4cd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303dda16d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303dab1550>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303d141630>, 1554986.477862919)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303dab25d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303ddb5990>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048977000>, 1554986.380514673)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303dd64c10>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303dba3a10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b55b380>, 1554986.537290595)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303dda1110>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056d686e0>, 1554987.851090759)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303ddb6010>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531038d5470>, 1554985.048225149)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303dba1190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303dadac90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056d6bb60>, 1554987.035892307)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303dce0c90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303e4a3ed0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bf1e660>, 1554987.526918765)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303da7a4d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303dd2a190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056d698d0>, 1554988.044125837)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303db64e50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303dd19e50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303e5dab10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056d69e80>, 1554987.913304246)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303dd25e10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048b278c0>, 1554988.005627512)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303de99410>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303db62a50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303de67d10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303d3552b0>, 1554987.233504439)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303db62990>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303d3549f0>, 1554987.88883105)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303de64910>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303debac10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303d356cf0>, 1554988.201386974)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303debb4d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303dc5a010>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303d3560b0>, 1554987.315892538)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303dc5af90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303dc52fd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303dfdc8d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303d356900>, 1554987.538157246)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303d349750>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303e296cd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303d356c80>, 1554987.933810683)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303e295690>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303e58fdd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303de7b790>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303dd5a650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303d355010>, 1554988.062505328)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303dfde390>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303d143690>, 1554989.854910846)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303df884d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303d3556a0>, 1554987.745084564)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303de79f50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056d69010>, 1554986.694119584)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303dd59610>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303df71150>\n",
      "Evaluating workflow:  36%|███▌      | 1067/3000 [03:28<05:59,  5.38it/s]Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303d03a820>, 1554989.684340752)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303df72790>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303e296f10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303d03be70>, 1554989.021139413)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303db1d650>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303e6de350>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303d038980>, 1554989.348623898)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303e54f110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303e45a390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303e3ad650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303d0380c0>, 1554989.537570896)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303e08e250>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fc7c8b40>, 1554989.467675896)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303e3bf190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303df70c90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fc7cb9a0>, 1554989.165252828)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303db0e150>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303e15f750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303d039b00>, 1554989.45252069)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303e17d190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303e227590>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302c6f8210>, 1554989.736107589)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303e226ad0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303e12add0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303e2eff90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303dfd1610>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303dcc5410>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303dfa4850>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303e388d10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303d03ac10>, 1554989.3248723)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303dfa6810>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303d142200>, 1554989.298309492)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303ddbe650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303d03a580>, 1554989.40146062)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303e38b7d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303d03b5b0>, 1554988.107347871)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303dfd3b50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302c6f87c0>, 1554989.543994483)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303e129c50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303d357ee0>, 1554990.542078751)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303e2efe50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303dd471d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303d142dd0>, 1554990.100299681)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303df91cd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304359e6d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303d355860>, 1554989.956142059)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304359e350>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530438c6910>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fc7ca890>, 1554990.709051446)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530438c67d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303e164350>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302c6fb770>, 1554991.103475677)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303e011210>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fc2ea690>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302c6fa5f0>, 1554990.954476207)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fc2ea450>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fce5cfd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fd3aeed0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bcb01a0>, 1554991.35238881)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fce5f010>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bcb0440>, 1554991.399559515)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fd3adf50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303e050b90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fcf6a9d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048b25cc0>, 1554991.230648462)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303e053f90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fcc5d590>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303d140600>, 1554991.589206987)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fcc5c590>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fccb6490>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fca0d590>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303e16cd10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303e241b50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153042c88c90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303d03a430>, 1554990.767177078)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303e16e650>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303dfa4110>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303d03a9e0>, 1554990.837775268)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303e1d7190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302c6f89f0>, 1554991.33250405)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303e240050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302c6f90f0>, 1554990.909132166)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153042c89190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bcb0ec0>, 1554991.768661077)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303e0b95d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bcb1860>, 1554991.173789943)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fccb7b10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bcb1a20>, 1554992.079842178)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fca0c850>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303e071c90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303d039710>, 1554991.917149956)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153042c5e890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153057936510>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bcb0750>, 1554991.613289313)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153057935390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fd4cc6d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302c6fbbd0>, 1554991.272435946)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fd4cdd50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303df3e0d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048b24f30>, 1554991.598303499)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303df3ed50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530438c7bd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303d03b380>, 1554992.769453395)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530438c61d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fc4b60d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153105036c80>, 1554992.487658594)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303e276f50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fd2e5f10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303dfc1390>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bcb2970>, 1554992.66088025)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fc6781d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bcb2f20>, 1554992.958626911)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fd606490>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fc20d4d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b9fc130>, 1554993.162892162)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fc20d6d0>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153100bdf510>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302c6fa7b0>, 1554993.284559)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153100bdfe90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fd436b50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b9fd320>, 1554993.018908845)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fd435c10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fe537910>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fceda210>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b9fd710>, 1554992.894772611)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fe494f50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302c6fb150>, 1554993.05071998)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fced8d50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fe423b50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fd668490>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bcb2200>, 1554991.488529255)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fd6692d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303d355390>, 1554993.304729261)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fe422cd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fd753110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fce6a490>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b9fe9e0>, 1554993.175452876)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fc8cc050>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fcd08390>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b9ff620>, 1554993.220012701)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fcd0a690>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fcce7c50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b9ffd20>, 1554993.398030875)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303e2a04d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fd1837d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fc5cef50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b9fff50>, 1554993.404029727)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fd180590>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056b041a0>, 1554993.499936113)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fc5ccf50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fccfaf50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fc4becd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303d355b00>, 1554993.364056288)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fccfb410>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fc7cb690>, 1554993.546568688)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fc3be6d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fc58a150>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056b04fa0>, 1554993.703258752)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fc58af50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153057aef290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056b05a20>, 1554994.701922025)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153057aecbd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153057ac1190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056b066d0>, 1554994.525462733)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153057ac39d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fc40b990>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153057a2b1d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fc4419d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303d3574d0>, 1554993.771626699)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fce69350>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fcdc1d90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056b06f20>, 1554994.630821921)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fc40ba50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b9fe430>, 1554993.10623178)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531010875d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303d039160>, 1554994.987491711)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153057a29910>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056b05d30>, 1554995.965776446)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fc443f50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fcb0fdd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd18cb40>, 1554995.66242459)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fd17de10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fd70fd90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd18cf30>, 1554994.827008693)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fd70d990>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fcf89450>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303d0388a0>, 1554995.48825274)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fcf8a850>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153056c0f650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd18da90>, 1554994.930681478)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fd4e6550>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fc998410>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fc632e10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd18e0b0>, 1554995.087546764)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fc99bb10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd18e2e0>, 1554995.361231173)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fc630090>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530579c1150>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153056d43990>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b9fdf60>, 1554995.365665581)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530579c1cd0>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd18ee40>, 1554995.53971945)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153056d41690>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fd1d1c10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd18f070>, 1554995.556480001)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fd24d410>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153056d7f890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fd1f2210>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd18c360>, 1554995.336448786)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153056d7c190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd18fa80>, 1554995.559723924)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fd2bb7d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fc38bed0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fca31a90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd18fd20>, 1554995.704238456)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fc388210>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fc0e5990>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530578766d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fc126cd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd18f540>, 1554995.303433094)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fcd42fd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fc9b47c0>, 1554996.307964787)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fc568dd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b9fd8d0>, 1554994.84298214)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153057876090>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd18d550>, 1554994.443323795)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fc124d50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531011d8650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fc9b48a0>, 1554996.48834074)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153056e7f350>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fd3c5c10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303d03b4d0>, 1554996.854528941)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fd3c7450>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fc4cc810>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fe5566d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fc9b5780>, 1554996.735773927)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153057970690>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fcb87910>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531077cd610>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fc9b6740>, 1554997.010858234)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fcb85c50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153107841890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bcb0d00>, 1555030.753875011)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153107841090>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531076dc550>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fd1cbed0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fd61b9d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056b04c90>, 1554997.176773088)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fd1c8fd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fcfc0de0>, 1554997.344067776)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fd61abd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153107843d10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15310763e350>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fcfc11d0>, 1555030.458190902)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531077c9610>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fcfc12b0>, 1554997.340561359)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15310761d890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531074f5250>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056b075b0>, 1554997.65224449)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531074b84d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fca8fc10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15310754f6d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fcfc1fd0>, 1555032.181203398)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fca8eb10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fcfc2270>, 1554997.22160495)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15310754f2d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15310779e690>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531073ee6d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fc9b42f0>, 1554996.503790213)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fc94b950>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd18f230>, 1555030.551767303)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531077cdbd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b9fec10>, 1554996.99264632)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531076c66d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fcfc2430>, 1554997.877352058)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15310779c790>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056d6b4d0>, 1554997.531653019)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531073eeb50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153057aa0cd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fcfc1860>, 1555032.047967088)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153057aa2710>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153107302750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fcfc3d20>, 1555032.364075809)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153107303d50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fe557d10>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fcc24440>, 1555032.347818665)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153107319890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531072e4a50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fcc246e0>, 1555030.816065923)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fd3c4d90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fcd09910>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153107250e10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fcc242f0>, 1555030.41877344)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153107218f90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fcc254e0>, 1555030.699351571)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531076de6d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531072ae790>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fcc26120>, 1555030.708835927)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531072ac310>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531072310d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fcc26350>, 1555032.226631661)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153107232dd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15310765ca90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531071a66d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fc9b7070>, 1555031.803446711)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15310714f550>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fcc26d60>, 1555032.167169486)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531071a7150>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153107006890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531070637d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fcc27540>, 1555032.016839955)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153107007b90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b9ffee0>, 1555031.888096467)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153107042010>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153107128f50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd18e740>, 1555032.441831485)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153107129890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153106efc1d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153107593bd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057d68130>, 1555031.812959022)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153107026850>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fcfc3230>, 1554997.331644562)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531075936d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153106ee0050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057d68360>, 1555033.65543119)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153106ee2ed0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153106c1d090>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fce75080>, 1555034.077244418)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153106c1cb90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153107573b90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153106da2110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153106adac50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531069e8c10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153106baa7d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153106a8b150>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153106945310>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057d687c0>, 1555032.199373868)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15310709fc50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fce74e50>, 1555033.233784597)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153106da0f10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fce75e80>, 1555033.472794364)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153106ada410>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fce766d0>, 1555033.679584285)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531069e9750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fce76900>, 1555034.310869129)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153106abe5d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fce77310>, 1555033.260310046)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153106a88e10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fce77d90>, 1555033.532196974)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15310692bdd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15310707c210>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fcfc2970>, 1555034.189061441)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15310707d390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153106d61910>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fce74bb0>, 1555034.525980057)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153106d61c10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531068565d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fcc7dc50>, 1555033.925698711)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531068560d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153106d61310>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fce74830>, 1555034.206934047)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153106d61a10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153106ee1a90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd18cd70>, 1555034.771250139)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153106ee27d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153106adb4d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153106562510>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b9ffbd0>, 1555034.747388485)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153106563810>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153106df2090>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531072afb10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15310681f5d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15310674df90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531065cee50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fc9b5240>, 1555035.433104002)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15310653da50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153106713e50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fce75a90>, 1555035.300847495)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15310681f1d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153106f568d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531067abed0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fcc7e120>, 1555034.666697251)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15310674c650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fcc7d8d0>, 1555033.805196037)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153106fccb10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fcc7d9b0>, 1555035.92371423)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15310678e6d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fce75da0>, 1555035.434418662)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fc6231d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fcfc3150>, 1555034.25917241)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531072af1d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fcc7cbb0>, 1555035.164138173)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531065cc590>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fcc7c6e0>, 1555035.078465369)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153106713a50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153106562f90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fcfc3c40>, 1555035.472064464)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153106561910>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153106312590>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304284a3c0>, 1555036.005025998)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153106312090>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15310638fb50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fcc7f8c0>, 1555036.072350525)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15310638e1d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531061cf990>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153106209190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304284aba0>, 1555035.680410249)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531061cf590>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fcfc0b40>, 1555036.202961815)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15310620b190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153106240210>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153042849e80>, 1555036.485251424)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153106241bd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153106331fd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15310611ca50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304284a5f0>, 1555036.262152711)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153106331dd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304284b7e0>, 1555036.339765726)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531060fc510>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531060c5110>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304284bf50>, 1555036.449651833)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15310615d4d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531072ada90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153105fd99d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fcad4210>, 1555036.702110805)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303dc108d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fcad44b0>, 1555036.804780629)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153106562190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153106074710>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153106015750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fcc7f540>, 1555037.022397458)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153106014f50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153105f2b210>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531067abf90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153106415190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fcc25010>, 1555037.157989793)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153106075890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531064702d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153042848bb0>, 1555035.413145256)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153106471dd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fcad5b70>, 1555037.166375282)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153105f2ad10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b9fc0c0>, 1555035.554266207)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531067abd90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153042849a90>, 1555035.61730673)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153106417e90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153105ed15d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153105dc7d50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fcfc0bb0>, 1555036.853914168)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153105ed11d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fcad67b0>, 1555037.357932221)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153105dc7850>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153105f64a50>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fcc27a10>, 1555037.3282928)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153105f64410>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153105f0dfd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153105ea3610>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fcad6d60>, 1555037.531738518)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153105f0da90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fcad7310>, 1555037.171615555)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153105ea3210>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153105d1a090>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fcc7ee40>, 1555037.251709294)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153105d1b490>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153105cc7350>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd4e0210>, 1555037.664253262)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153105cc6dd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15310611e810>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd4e0910>, 1555037.665826096)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153106bf84d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531060c6390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153105c90190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304284bcb0>, 1555037.81654228)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15310617d690>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153105cabc10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd4e1780>, 1555038.559516233)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153105c93f10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153105dac350>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153105b44f10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd4e1b00>, 1555037.923914671)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531062cf610>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fcc24bb0>, 1555038.24931082)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153105b471d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153105ba0610>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd4e2890>, 1555038.345543974)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153105b81b10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531059dfc90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153105f481d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304284aac0>, 1555037.673733393)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153105c6ff50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fcfc1a20>, 1555038.538580565)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153106b91190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fcad6200>, 1555036.983376032)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153105f4b010>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153105a9cc50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057d6b7e0>, 1555038.515121567)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153105a9eb50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153105a9f0d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd4e3b60>, 1555038.968436721)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153106090c50>\n",
      "Evaluating workflow:  36%|███▌      | 1071/3000 [03:28<03:44,  8.61it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  36%|███▌      | 1078/3000 [03:28<02:08, 14.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  36%|███▌      | 1081/3000 [03:28<02:05, 15.29it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  36%|███▌      | 1085/3000 [04:01<1:17:53,  2.44s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  36%|███▋      | 1091/3000 [04:02<40:14,  1.26s/it]  "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  36%|███▋      | 1095/3000 [04:03<25:01,  1.27it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  37%|███▋      | 1099/3000 [04:04<16:34,  1.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  37%|███▋      | 1102/3000 [04:04<11:30,  2.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  37%|███▋      | 1104/3000 [04:04<09:08,  3.46it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  37%|███▋      | 1109/3000 [04:05<05:50,  5.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  37%|███▋      | 1111/3000 [04:05<05:27,  5.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  37%|███▋      | 1115/3000 [04:05<04:33,  6.88it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  37%|███▋      | 1119/3000 [04:06<03:46,  8.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  37%|███▋      | 1121/3000 [04:06<03:07, 10.02it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  38%|███▊      | 1125/3000 [04:06<02:40, 11.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  38%|███▊      | 1127/3000 [04:07<02:57, 10.55it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  38%|███▊      | 1129/3000 [04:07<02:49, 11.06it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  38%|███▊      | 1131/3000 [04:07<04:04,  7.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  38%|███▊      | 1133/3000 [04:07<04:37,  6.72it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  38%|███▊      | 1136/3000 [04:08<03:56,  7.88it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  38%|███▊      | 1142/3000 [04:08<02:08, 14.42it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  38%|███▊      | 1144/3000 [04:08<02:02, 15.14it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  38%|███▊      | 1147/3000 [04:09<03:10,  9.72it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  38%|███▊      | 1152/3000 [04:09<03:19,  9.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  38%|███▊      | 1154/3000 [04:09<02:58, 10.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  39%|███▊      | 1158/3000 [04:10<02:57, 10.35it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  39%|███▉      | 1163/3000 [04:10<02:14, 13.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  39%|███▉      | 1165/3000 [04:10<02:12, 13.87it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  39%|███▉      | 1169/3000 [04:10<02:09, 14.09it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  39%|███▉      | 1171/3000 [04:11<03:20,  9.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  39%|███▉      | 1173/3000 [04:11<04:10,  7.29it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  39%|███▉      | 1176/3000 [04:11<03:23,  8.96it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  39%|███▉      | 1178/3000 [04:12<03:06,  9.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  39%|███▉      | 1182/3000 [04:12<02:45, 11.02it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  39%|███▉      | 1184/3000 [04:12<02:39, 11.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  40%|███▉      | 1189/3000 [04:13<02:37, 11.47it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  40%|███▉      | 1191/3000 [04:13<02:32, 11.82it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  40%|███▉      | 1193/3000 [04:13<03:48,  7.89it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  40%|███▉      | 1199/3000 [04:14<02:40, 11.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  40%|████      | 1201/3000 [04:14<02:23, 12.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  40%|████      | 1205/3000 [04:14<02:22, 12.58it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  40%|████      | 1207/3000 [04:14<02:28, 12.04it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  40%|████      | 1209/3000 [04:14<02:43, 10.93it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  40%|████      | 1211/3000 [04:15<03:00,  9.93it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  40%|████      | 1215/3000 [04:15<03:06,  9.55it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  41%|████      | 1217/3000 [04:15<02:44, 10.87it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  41%|████      | 1219/3000 [04:16<03:12,  9.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  41%|████      | 1225/3000 [04:16<02:03, 14.42it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  41%|████      | 1229/3000 [04:16<02:32, 11.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  41%|████      | 1231/3000 [04:16<02:23, 12.32it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  41%|████      | 1233/3000 [04:17<02:27, 11.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  41%|████      | 1235/3000 [04:17<04:02,  7.29it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  41%|████      | 1237/3000 [04:17<03:27,  8.48it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  41%|████▏     | 1239/3000 [04:18<03:10,  9.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  41%|████▏     | 1241/3000 [04:18<02:56,  9.96it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  42%|████▏     | 1247/3000 [04:18<02:10, 13.40it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  42%|████▏     | 1249/3000 [04:18<03:02,  9.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  42%|████▏     | 1251/3000 [04:19<03:53,  7.49it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  42%|████▏     | 1256/3000 [04:19<02:43, 10.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  42%|████▏     | 1258/3000 [04:19<03:18,  8.77it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  42%|████▏     | 1262/3000 [04:20<04:22,  6.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  42%|████▏     | 1264/3000 [04:21<03:49,  7.55it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  42%|████▏     | 1266/3000 [04:21<03:24,  8.46it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  42%|████▏     | 1272/3000 [04:21<02:26, 11.82it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  42%|████▏     | 1274/3000 [04:22<04:10,  6.89it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  43%|████▎     | 1280/3000 [04:22<02:53,  9.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  43%|████▎     | 1282/3000 [04:22<02:45, 10.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  43%|████▎     | 1287/3000 [04:23<02:25, 11.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  43%|████▎     | 1289/3000 [04:23<02:11, 12.96it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  43%|████▎     | 1293/3000 [04:23<02:53,  9.83it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  43%|████▎     | 1296/3000 [04:24<03:54,  7.27it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  43%|████▎     | 1299/3000 [04:24<02:58,  9.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  43%|████▎     | 1304/3000 [04:25<02:26, 11.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  44%|████▎     | 1307/3000 [04:25<02:03, 13.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  44%|████▎     | 1310/3000 [04:25<02:03, 13.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  44%|████▎     | 1312/3000 [04:25<02:39, 10.57it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  44%|████▍     | 1316/3000 [04:26<02:50,  9.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  44%|████▍     | 1319/3000 [04:26<02:44, 10.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  44%|████▍     | 1324/3000 [04:26<02:29, 11.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  44%|████▍     | 1326/3000 [04:27<02:33, 10.93it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  44%|████▍     | 1332/3000 [04:27<02:10, 12.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  44%|████▍     | 1334/3000 [04:27<02:13, 12.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  45%|████▍     | 1336/3000 [04:27<02:35, 10.71it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  45%|████▍     | 1340/3000 [04:28<03:03,  9.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  45%|████▍     | 1342/3000 [04:28<02:51,  9.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  45%|████▍     | 1347/3000 [04:29<02:26, 11.29it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  45%|████▌     | 1351/3000 [04:29<02:13, 12.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  45%|████▌     | 1355/3000 [05:01<1:26:52,  3.17s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "\u001b[32m2026-01-04 10:49:53.576\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'question': \"You are a functional genomics specialist. In K562 cells, we perturb SNAPC1 and monitor IMPA2 expression. Decide whether this perturbation leads to a significant alteration in IMPA2 expression. Answer only in the format 'Final Answer: Yes' or 'Final Answer: No'.\", 'answer': 'Yes', 'question_new': \"You are a functional genomics specialist. In K562 cells, we perturb SNAPC1 and monitor IMPA2 expression. Decide whether this perturbation leads to a significant alteration in IMPA2 expression. Answer only in the format 'Final Answer: Yes' or 'Final Answer: No'.\", '_id': 49548}\n",
      "Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 1 second. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  45%|████▌     | 1357/3000 [05:02<1:04:44,  2.36s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2026-01-04 10:49:53.590\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'question': \"You are an expert in single-cell biology and functional genomics. In K562 cells, HSPH1 is perturbed and the expression of FBXO5 is measured. Does this perturbation cause a significant change in FBXO5 expression? Answer strictly in the format 'Final Answer: Yes' or 'Final Answer: No'.\", 'answer': 'Yes', 'question_new': \"You are an expert in single-cell biology and functional genomics. In K562 cells, HSPH1 is perturbed and the expression of FBXO5 is measured. Does this perturbation cause a significant change in FBXO5 expression? Answer strictly in the format 'Final Answer: Yes' or 'Final Answer: No'.\", '_id': 20990}\n",
      "Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 1 second. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n",
      "\u001b[32m2026-01-04 10:49:53.602\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'question': \"You are an expert in single-cell biology and functional genomics. In K562 cells, NSL1 is perturbed and the expression of NSL1 is measured. Does this perturbation cause a significant change in NSL1 expression? Answer strictly in the format 'Final Answer: Yes' or 'Final Answer: No'.\", 'answer': 'Yes', 'question_new': \"You are an expert in single-cell biology and functional genomics. In K562 cells, NSL1 is perturbed and the expression of NSL1 is measured. Does this perturbation cause a significant change in NSL1 expression? Answer strictly in the format 'Final Answer: Yes' or 'Final Answer: No'.\", '_id': 32550}\n",
      "Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 1 second. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n",
      "\u001b[32m2026-01-04 10:49:53.663\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'question': \"You are an expert analyst of perturbation datasets. For K562 cells subjected to perturbation of EXOSC3, does the expression profile of PCLAF indicate a significant change relative to control conditions? Reply strictly with 'Final Answer: Yes' or 'Final Answer: No'.\", 'answer': 'Yes', 'question_new': \"You are an expert analyst of perturbation datasets. For K562 cells subjected to perturbation of EXOSC3, does the expression profile of PCLAF indicate a significant change relative to control conditions? Reply strictly with 'Final Answer: Yes' or 'Final Answer: No'.\", '_id': 16077}\n",
      "Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 1 second. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n",
      "\u001b[32m2026-01-04 10:49:53.677\u001b[0m | \u001b[31m\u001b[1mERROR   \u001b[0m | \u001b[36mevoagentx.workflow.workflow\u001b[0m:\u001b[36masync_execute\u001b[0m:\u001b[36m104\u001b[0m - \u001b[31m\u001b[1mAn Error occurs when executing the workflow: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 1 second. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  45%|████▌     | 1361/3000 [05:02<36:02,  1.32s/it]  "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "\u001b[32m2026-01-04 10:49:53.722\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'question': \"You are a domain expert in functional genomics. For experiments carried out in K562 cells, we perturb GPN3 and examine the expression of AFF3. Does perturbing GPN3 lead to a significant change in AFF3 expression? Reply only with 'Final Answer: Yes' or 'Final Answer: No'.\", 'answer': 'Yes', 'question_new': \"You are a domain expert in functional genomics. For experiments carried out in K562 cells, we perturb GPN3 and examine the expression of AFF3. Does perturbing GPN3 lead to a significant change in AFF3 expression? Reply only with 'Final Answer: Yes' or 'Final Answer: No'.\", '_id': 18501}\n",
      "Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 1 second. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  46%|████▌     | 1365/3000 [05:03<21:54,  1.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  46%|████▌     | 1367/3000 [05:03<16:39,  1.63it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  46%|████▌     | 1371/3000 [05:03<10:05,  2.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  46%|████▌     | 1373/3000 [05:04<10:41,  2.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  46%|████▌     | 1378/3000 [05:04<06:17,  4.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  46%|████▌     | 1381/3000 [05:05<05:20,  5.06it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  46%|████▌     | 1384/3000 [05:05<05:49,  4.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  46%|████▌     | 1386/3000 [05:06<04:30,  5.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  46%|████▋     | 1390/3000 [05:06<03:36,  7.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  46%|████▋     | 1393/3000 [05:06<02:33, 10.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  47%|████▋     | 1398/3000 [05:06<02:01, 13.14it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  47%|████▋     | 1400/3000 [05:07<01:57, 13.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  47%|████▋     | 1402/3000 [05:07<03:07,  8.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  47%|████▋     | 1404/3000 [05:07<02:49,  9.40it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  47%|████▋     | 1408/3000 [05:08<02:48,  9.47it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  47%|████▋     | 1410/3000 [05:08<02:54,  9.14it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  47%|████▋     | 1414/3000 [05:08<02:22, 11.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  47%|████▋     | 1418/3000 [05:08<02:13, 11.87it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  47%|████▋     | 1420/3000 [05:09<03:03,  8.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  47%|████▋     | 1424/3000 [05:09<02:40,  9.83it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  48%|████▊     | 1428/3000 [05:09<02:03, 12.78it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  48%|████▊     | 1431/3000 [05:10<02:13, 11.78it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  48%|████▊     | 1434/3000 [05:10<03:28,  7.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  48%|████▊     | 1439/3000 [05:11<02:47,  9.32it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  48%|████▊     | 1441/3000 [05:11<02:41,  9.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  48%|████▊     | 1444/3000 [05:11<02:06, 12.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  48%|████▊     | 1446/3000 [05:11<02:27, 10.55it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  48%|████▊     | 1451/3000 [05:12<02:20, 11.04it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  48%|████▊     | 1453/3000 [05:12<02:35,  9.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  49%|████▊     | 1456/3000 [05:13<02:57,  8.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  49%|████▊     | 1461/3000 [05:13<02:07, 12.04it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  49%|████▉     | 1463/3000 [05:13<02:07, 12.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  49%|████▉     | 1466/3000 [05:13<02:09, 11.86it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  49%|████▉     | 1469/3000 [05:14<02:02, 12.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  49%|████▉     | 1471/3000 [05:14<02:38,  9.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  49%|████▉     | 1475/3000 [05:14<02:53,  8.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  49%|████▉     | 1477/3000 [05:14<02:27, 10.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  49%|████▉     | 1482/3000 [05:15<02:08, 11.84it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  49%|████▉     | 1484/3000 [05:15<01:54, 13.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  50%|████▉     | 1488/3000 [05:15<02:10, 11.55it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  50%|████▉     | 1490/3000 [05:16<02:04, 12.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  50%|████▉     | 1492/3000 [05:16<02:11, 11.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  50%|████▉     | 1497/3000 [05:17<02:54,  8.61it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  50%|█████     | 1501/3000 [05:17<02:16, 10.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  50%|█████     | 1504/3000 [05:17<01:58, 12.66it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  50%|█████     | 1507/3000 [05:17<01:46, 14.06it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  50%|█████     | 1509/3000 [05:17<02:19, 10.67it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  50%|█████     | 1513/3000 [05:18<02:45,  9.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  51%|█████     | 1517/3000 [05:18<02:22, 10.38it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  51%|█████     | 1520/3000 [05:18<01:48, 13.61it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  51%|█████     | 1522/3000 [05:19<02:16, 10.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  51%|█████     | 1524/3000 [05:19<02:09, 11.42it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  51%|█████     | 1527/3000 [05:19<02:01, 12.11it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  51%|█████     | 1529/3000 [05:20<02:52,  8.52it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  51%|█████     | 1533/3000 [05:20<02:41,  9.07it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  51%|█████     | 1536/3000 [05:20<02:01, 12.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  51%|█████▏    | 1540/3000 [05:21<02:32,  9.55it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  51%|█████▏    | 1542/3000 [05:21<03:29,  6.97it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  52%|█████▏    | 1546/3000 [05:22<02:49,  8.57it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  52%|█████▏    | 1548/3000 [05:22<03:21,  7.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  52%|█████▏    | 1551/3000 [05:22<03:20,  7.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  52%|█████▏    | 1555/3000 [05:23<02:30,  9.60it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  52%|█████▏    | 1559/3000 [05:23<02:02, 11.78it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  52%|█████▏    | 1562/3000 [05:23<01:46, 13.48it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  52%|█████▏    | 1564/3000 [05:23<01:44, 13.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  52%|█████▏    | 1566/3000 [05:24<03:01,  7.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  52%|█████▏    | 1568/3000 [05:24<02:55,  8.14it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  52%|█████▏    | 1572/3000 [05:24<02:32,  9.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  52%|█████▎    | 1575/3000 [05:25<01:56, 12.27it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  53%|█████▎    | 1580/3000 [05:25<01:41, 13.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  53%|█████▎    | 1582/3000 [05:25<01:58, 11.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  53%|█████▎    | 1586/3000 [05:26<02:06, 11.16it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  53%|█████▎    | 1588/3000 [05:26<02:49,  8.35it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  53%|█████▎    | 1592/3000 [05:26<02:20, 10.05it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  53%|█████▎    | 1596/3000 [05:27<02:12, 10.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  53%|█████▎    | 1599/3000 [05:27<01:48, 12.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  53%|█████▎    | 1601/3000 [05:27<02:00, 11.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  54%|█████▎    | 1605/3000 [05:27<01:58, 11.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  54%|█████▎    | 1607/3000 [05:28<02:04, 11.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  54%|█████▎    | 1609/3000 [05:28<02:13, 10.46it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  54%|█████▍    | 1613/3000 [05:28<02:36,  8.88it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  54%|█████▍    | 1615/3000 [05:28<02:12, 10.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  54%|█████▍    | 1617/3000 [05:29<02:52,  8.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  54%|█████▍    | 1620/3000 [05:29<03:08,  7.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  54%|█████▍    | 1625/3000 [05:30<02:00, 11.40it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  54%|█████▍    | 1628/3000 [06:01<1:27:12,  3.81s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  54%|█████▍    | 1630/3000 [06:01<59:08,  2.59s/it]  "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  54%|█████▍    | 1632/3000 [06:02<42:05,  1.85s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  54%|█████▍    | 1634/3000 [06:03<31:20,  1.38s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "\u001b[32m2026-01-04 10:50:54.413\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'question': \"You are an expert analyst of perturbation datasets. For K562 cells subjected to perturbation of AAR2, does the expression profile of PLEKHA5 indicate a significant change relative to control conditions? Reply strictly with 'Final Answer: Yes' or 'Final Answer: No'.\", 'answer': 'Yes', 'question_new': \"You are an expert analyst of perturbation datasets. For K562 cells subjected to perturbation of AAR2, does the expression profile of PLEKHA5 indicate a significant change relative to control conditions? Reply strictly with 'Final Answer: Yes' or 'Final Answer: No'.\", '_id': 67}\n",
      "Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 1 second. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n",
      "\u001b[32m2026-01-04 10:50:54.505\u001b[0m | \u001b[31m\u001b[1mERROR   \u001b[0m | \u001b[36mevoagentx.workflow.workflow\u001b[0m:\u001b[36masync_execute\u001b[0m:\u001b[36m104\u001b[0m - \u001b[31m\u001b[1mAn Error occurs when executing the workflow: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 1 second. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  55%|█████▍    | 1637/3000 [06:03<19:14,  1.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  55%|█████▍    | 1642/3000 [06:03<10:02,  2.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  55%|█████▍    | 1645/3000 [06:04<09:26,  2.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  55%|█████▍    | 1648/3000 [06:04<05:53,  3.82it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  55%|█████▌    | 1653/3000 [06:05<03:44,  6.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  55%|█████▌    | 1658/3000 [06:06<03:19,  6.74it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  55%|█████▌    | 1660/3000 [06:06<02:59,  7.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  55%|█████▌    | 1662/3000 [06:06<02:46,  8.05it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  55%|█████▌    | 1664/3000 [06:06<02:35,  8.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  56%|█████▌    | 1668/3000 [06:07<02:31,  8.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  56%|█████▌    | 1670/3000 [06:07<02:20,  9.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  56%|█████▌    | 1672/3000 [06:07<02:18,  9.61it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  56%|█████▌    | 1676/3000 [06:08<02:46,  7.97it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  56%|█████▌    | 1678/3000 [06:08<02:59,  7.36it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  56%|█████▌    | 1684/3000 [06:09<02:00, 10.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  56%|█████▌    | 1686/3000 [06:09<02:01, 10.83it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  56%|█████▋    | 1691/3000 [06:09<02:30,  8.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  56%|█████▋    | 1693/3000 [06:10<02:17,  9.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  57%|█████▋    | 1697/3000 [06:10<01:57, 11.07it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  57%|█████▋    | 1699/3000 [06:10<02:17,  9.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  57%|█████▋    | 1703/3000 [06:11<02:16,  9.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  57%|█████▋    | 1705/3000 [06:11<02:14,  9.63it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  57%|█████▋    | 1709/3000 [06:11<01:47, 12.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  57%|█████▋    | 1711/3000 [06:11<02:24,  8.89it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  57%|█████▋    | 1715/3000 [06:12<02:08,  9.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  57%|█████▋    | 1719/3000 [06:12<02:07, 10.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  57%|█████▋    | 1723/3000 [06:12<01:38, 12.97it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  58%|█████▊    | 1726/3000 [06:13<01:24, 15.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  58%|█████▊    | 1730/3000 [06:13<01:40, 12.67it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  58%|█████▊    | 1734/3000 [06:14<02:57,  7.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  58%|█████▊    | 1736/3000 [06:14<02:30,  8.38it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153105950250>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057f08670>, 1555039.321044539)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153105a3b990>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153105b63050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057f08fa0>, 1555038.928161454)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531057cead0>\n",
      "Evaluating workflow:  58%|█████▊    | 1739/3000 [06:15<04:03,  5.18it/s]Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153105827190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057f09390>, 1555038.51907964)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153105826c90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153105ba35d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057c0e040>, 1555040.238510785)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153105ba20d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153102bc4250>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fc914130>, 1555041.264939998)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15310619ffd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531059dcb10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057c0d320>, 1555039.780691516)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531059dfcd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531028c9f90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fc916e40>, 1555041.706552393)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153102641090>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531026b8e10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057ea49f0>, 1555042.28616622)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153105ea2590>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153102381cd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057ea5b70>, 1555042.385469944)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531023824d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153102a46250>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057ea5710>, 1555042.64642327)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531024866d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153102979350>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057734130>, 1555042.964939854)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531029f6ed0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531020634d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057734ec0>, 1555042.990579506)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153102063fd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153101eed090>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057737770>, 1555043.884281855)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153101eed890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531021e86d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057734360>, 1555042.777140939)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153102136b50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d7f1a50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530575f5780>, 1555045.422612196)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d7f1d50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153100b71510>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153100b74ad0>, 1555046.523327978)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153100b71750>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304b1f9dd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153100b77d90>, 1555047.012314336)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304b1f0050>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302c7a8310>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153101b62150>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530575f6660>, 1555046.850116267)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302c797bd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057097620>, 1555050.027890591)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153101b62050>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153107867350>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302c78f230>, 1555051.191840781)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531078675d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531063c3050>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302c781b10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153101bbf1d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153101b6d790>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531040180c0>, 1555050.026174031)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304b1a2490>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15310401b620>, 1555048.668042211)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153101b404d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302c78c670>, 1555048.612928575)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153101bbf550>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531078a8590>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302c78eeb0>, 1555051.575815059)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531063c2f50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153107809940>, 1555051.478718451)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531078a8390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531063e9210>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153106404130>, 1555051.435154459)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531063e9410>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15310643e650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153106405400>, 1555051.954659374)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15310643e450>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531078643d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153106406dd0>, 1555051.595259092)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153107864610>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153105cc9550>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153105ce1940>, 1555051.753181643)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153105cc9610>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153105ce4e50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15310640a390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153106498c90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153106404d70>, 1555051.328701354)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15310640a190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153105ce2120>, 1555051.784032089)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153105d9b4d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15310780a2e0>, 1555051.105939171)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153106499390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153106426050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153105ce3e70>, 1555053.289956995)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153106424dd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153106448c10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15310780aa50>, 1555053.83472089)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153106448e10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531028a8510>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531028f97f0>, 1555053.621326402)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531028a8450>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531021b9050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302c78de80>, 1555053.96370803)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531021b8f50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531021e4910>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531028faeb0>, 1555053.537721503)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153102913b90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531057fd4d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531057d0b40>, 1555055.133153531)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531057fefd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153102204c90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531021d0850>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531028fb070>, 1555053.619879647)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153102204a90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531028fb770>, 1555053.795748439)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531021d0650>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153107638550>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531078a89d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153107614ec0>, 1555055.507681335)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153107638410>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153107614c90>, 1555055.64233726)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15310583ef90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153107656090>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153107616200>, 1555055.437648981)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153107655f50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153107633950>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153107616ba0>, 1555056.121008351)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153107633110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531058324d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153107561010>, 1555057.597217872)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531075066d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153107365890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153107390950>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531075635b0>, 1555057.760173515)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153107365b10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153107614590>, 1555057.761786958)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153107390750>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15310217bbd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531072c9310>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153107563d20>, 1555057.753419614)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153107626250>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153106404c20>, 1555057.777038568)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531072c9110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153106fc11d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153107354950>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153106fd6a90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531073348a0>, 1555057.65894874)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153107355510>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531057d3f50>, 1555057.747946393)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153106fd6950>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153107307950>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153107016ad0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531073351d0>, 1555058.040687481)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153107306890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153107335860>, 1555057.590732468)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153107016fd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153107029610>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153107336120>, 1555057.772214888)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531070294d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153107696690>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153107336510>, 1555057.937111514)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153107696090>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15310707b090>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531076152b0>, 1555058.105839962)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531070796d0>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15310705bb10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531073047d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153107334a60>, 1555057.593604031)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153106fc16d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153107337230>, 1555092.692279849)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153107091710>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531076174d0>, 1555056.507875321)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531073045d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153106ff5790>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153106cc3190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153105913190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15310596c110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531058a3d10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531056c0750>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153105d93e50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153105bd0210>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153105c6db90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15310596db90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531058bbf10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531055a00d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531053e18d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153102cac890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153106606650>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153102a5ed90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153102af5d50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153102a7fed0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531029b64d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531056c11d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531052bce10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153102c72610>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531054a0bd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531029f69d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153102889810>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153102820c50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153102577710>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531024aa790>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531024e1b10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153102363690>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153102baa190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153102802410>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153102714650>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531055a0050>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531023f8f90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531022d2810>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153102822ed0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531020a0190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153102100710>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153101f63250>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153102087a10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153101e5ad50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153102101390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153105ba2ad0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153101f2fa10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153101e22410>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153101fb7450>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531021eb850>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153101cfe350>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153101d1aed0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153101b3e6d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153100bb0610>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531027c6c10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fe3ba210>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153100bb18d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302c7280d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302c702250>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531057d8c90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153101bb7850>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153102102290>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302c70ce90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153101b3e0d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302c6ed4d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153102382950>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153100b49b50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153100b5f090>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153100bb4750>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531023d4f10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304b138710>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153100b4afd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304b1c7910>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153101ca2110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304b1e8a50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304b1f3f50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302c771210>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153102117050>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302c7c0610>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302c7579d0>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302c7d9b10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153100b70510>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531040ef710>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531040b4b50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531040bc510>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531040c2b90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531040dc450>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531040254d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302c781310>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153101bee510>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15310781fe10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15310786df50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304b101450>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153107892a50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15310785d4d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531077e66d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531063c0710>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153101b43f10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531063eb150>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153106482b10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153105cfde50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15310787c450>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153105d41110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153105d8d990>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153106481390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153105dab0d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153105d6a790>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153101b684d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153102863610>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153106466ad0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531028bc350>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153102910710>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153102841010>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153102887b50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153102159510>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153105cc8790>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153105d8e550>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531021d9d10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15310644b110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15310221c1d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153102839650>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153105834490>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531057ec050>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15310586f490>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531058a49d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531075f5d10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153105868490>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531075dc8d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153107604850>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153107666dd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153107694ed0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531076abe10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531074bee90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531074f8b10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531057ede50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15310755d4d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153107589510>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15310580c190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531072bda90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153107329cd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531057d25f0>, 1555057.333441701)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15310755d890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153107561400>, 1555057.339832795)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531072bd890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531073369e0>, 1555092.578063752)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153106ff51d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153106cd4210>, 1555091.044639138)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153106cc2fd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fcad41a0>, 1555038.884123504)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153105910c90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057f091d0>, 1555038.9556812)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15310596fa10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fcad56a0>, 1555039.181110281)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531058a3910>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057f08a60>, 1555039.489640429)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531056c1410>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057f0b2a0>, 1555039.984726728)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153105e01510>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057f0b700>, 1555039.747133334)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153105650690>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057f0b8c0>, 1555039.596665507)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153105c6ef90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057f0ba80>, 1555039.076379023)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153105512710>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057f08d70>, 1555039.483447193)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531058bbd10>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fcc7f4d0>, 1555039.97966466)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531055a0cd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057c0c520>, 1555039.876851404)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531053e11d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057c0e740>, 1555040.447538427)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153102cae290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057c0c7c0>, 1555040.679499949)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153105a9d8d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd4e2ac0>, 1555041.126360828)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531061eb150>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fc9142f0>, 1555040.830638778)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153102af5750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fcad4f30>, 1555040.844814025)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153102a7eb50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fc914bb0>, 1555041.018563179)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531029b5fd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153042849860>, 1555041.145011358)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531056c1750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd4e3770>, 1555040.182991572)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531052be990>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057c0e200>, 1555040.336261987)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153102c73c10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057c0ea50>, 1555040.413506028)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531054a23d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fc915e80>, 1555041.402235876)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531029f6fd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fcc7ed60>, 1555041.588775038)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15310288a590>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd4e0750>, 1555041.957369317)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153102820450>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057ea4130>, 1555041.940674596)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153102576f10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057ea5010>, 1555042.333697123)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531024aa290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057ea5240>, 1555042.01375434)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531024e1610>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd4e1ef0>, 1555042.52170598)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153102363290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057c0f690>, 1555043.160283734)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531029b6dd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fcad6e40>, 1555041.401007754)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153102803d10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057c0d710>, 1555041.917031464)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153102715350>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057ea42f0>, 1555041.912722449)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531055a3ed0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057ea6eb0>, 1555042.839599059)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531023faa10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057ea7770>, 1555042.887813962)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153102b17a90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fc914520>, 1555043.294247783)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153102252910>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057734fa0>, 1555043.41892136)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531020a13d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fc917c40>, 1555043.526365946)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153102103f90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057736900>, 1555043.564103506)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153101ff2f90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057736ac0>, 1555044.023347813)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153102087310>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057736d60>, 1555043.788537805)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15310202d890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057737150>, 1555043.567870924)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153102103ed0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057c0ce50>, 1555044.479868102)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153101f61050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fc914910>, 1555044.132234904)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153101f2d4d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fc915240>, 1555044.373558022)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153101e21c10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530577347c0>, 1555043.188055971)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153101fb5d50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057095240>, 1555044.601508954)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531021ea5d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057095940>, 1555045.016601007)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153101cffc50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057096120>, 1555044.809390492)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153101d1b8d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057096970>, 1555044.723468979)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153101b3d750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057095cc0>, 1555044.870775661)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531024c54d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057095f60>, 1555045.099982271)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531021c4510>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057097e70>, 1555045.271957697)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531052faa10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530575f4980>, 1555045.302485131)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153100bb20d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057ea7a10>, 1555045.375003857)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531020e1ed0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530575f55c0>, 1555045.288932604)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302c702490>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530575f4de0>, 1555045.804228808)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302c720410>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057ea60b0>, 1555045.011060611)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153101bb7050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057096f90>, 1555044.753676203)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531074d84d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057736740>, 1555045.294580223)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302c70e090>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530575f4ec0>, 1555045.770160851)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302c735390>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530575f6dd0>, 1555046.259927719)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302c6ef710>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530575f7000>, 1555046.047261409)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153101e23210>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530575f7930>, 1555046.36527197)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153100b4a0d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153100b741a0>, 1555046.411795677)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153100b5ee90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057097690>, 1555046.776715075)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302c6ecc90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057ea5d30>, 1555046.801058574)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531023d7c10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fc917460>, 1555046.710030671)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153100b48d10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153100b75be0>, 1555047.671393256)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153100b4bbd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153100b74750>, 1555048.311646065)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304b1c7710>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153100b76a50>, 1555046.785881426)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153101bd6f50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153100b74d70>, 1555047.055167836)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304b1e9110>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fc915be0>, 1555047.111646624)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304b1f3e50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153100b77ee0>, 1555048.234036088)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302c771010>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302c78c130>, 1555047.100147947)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153102117f50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530575f6120>, 1555048.517809015)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302c7b7f50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302c78c980>, 1555048.436892783)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302c756a50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302c78e270>, 1555048.565690259)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302c7d9fd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302c78e5f0>, 1555047.959389843)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153100b70550>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530575f77e0>, 1555047.946734296)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531040efbd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302c78f000>, 1555048.367973577)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531040b4a10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057ea6e40>, 1555048.471008412)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531040bc310>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153100b751d0>, 1555048.561605226)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153100b4b050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153100b773f0>, 1555049.941580966)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531040dc950>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153104019160>, 1555049.760282974)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153100b5d550>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153104019320>, 1555048.539694173)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304b1f3c10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15310401b3f0>, 1555049.990618828)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153101beec10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530575f69e0>, 1555049.523412416)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15310781fc10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302c78d2b0>, 1555049.558314177)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15310786de10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153107808d70>, 1555049.721906788)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531040406d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15310780a040>, 1555049.942666871)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153107892850>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15310780a890>, 1555050.088157911)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15310785f890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15310780aeb0>, 1555049.826789621)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531077e7410>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15310780bbd0>, 1555051.687779695)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531063c05d0>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15310401af20>, 1555049.752437897)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153101b43e90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531078083d0>, 1555051.789234053)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531063eba50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057f08c20>, 1555051.540103763)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531064832d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153105ce0b40>, 1555051.758654606)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153105cfdd10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153105ce0d00>, 1555051.422107948)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15310787d050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153100b76900>, 1555051.619024799)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153106499150>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302c78fe00>, 1555051.686733177)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153105d8e090>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153105ce1780>, 1555051.928709367)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153105d5d890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153105ce1fd0>, 1555051.587827914)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153105daaf90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153105ce27b0>, 1555052.062457975)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153105d6ae90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531078080c0>, 1555053.130861766)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153101b6a610>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153105ce0210>, 1555053.09215488)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153102863410>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15310401ac80>, 1555053.169806056)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153106467110>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153106404590>, 1555052.936881155)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531028bc1d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15310401b380>, 1555053.464165451)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153106448cd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153105ce3070>, 1555052.907586984)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153102840f10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531028f9240>, 1555053.603057271)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153102887a10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531028f9550>, 1555053.61849589)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153102159290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531028f9e80>, 1555053.704456181)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153105cc8510>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531028fa120>, 1555053.592883456)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153105d8f250>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531028fac10>, 1555053.22490511)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531064a5bd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531028fb540>, 1555053.994806342)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15310283b710>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531028f8d00>, 1555053.941315062)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15310221d350>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531057d1010>, 1555054.83030154)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153105cca590>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531028fbd90>, 1555054.834320656)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153105834b50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531057d1be0>, 1555054.775331511)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531057efd90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531057d2740>, 1555054.798423853)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15310586f350>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531028f8f30>, 1555055.116608133)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15310219ca10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531028f8de0>, 1555055.647336477)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531075f61d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531057d00c0>, 1555054.532953101)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153105868bd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531028f88a0>, 1555054.852535522)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531075dc6d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531028f90f0>, 1555055.203021608)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153107606e10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153107615080>, 1555055.468942307)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531058154d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15310780b850>, 1555055.857792414)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153107694cd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153107615ef0>, 1555055.591423989)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531076abd10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153107614440>, 1555055.589426674)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531074bf390>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153107616e40>, 1555055.718556058)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531074f89d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153107617ee0>, 1555056.315950826)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531057eda50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153105ce31c0>, 1555056.594121904)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531057fc410>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531075611d0>, 1555056.406564896)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15310580d1d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153107562b30>, 1555058.11818026)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15310732a310>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302c789050>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153107624ad0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531072f4650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302c78ca60>, 1555047.086993775)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302c78aa90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153107614980>, 1555055.33245586)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153107624a50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153107562190>, 1555057.847165095)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15310769cb90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153106d72f90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153106b69010>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153106d58d10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531058a5a50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057d6a660>, 1555092.030021317)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153106d585d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153106d366d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153106d284d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153102861950>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057d69550>, 1555092.49688238)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153106d37590>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057d69630>, 1555090.85620067)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153106b0b710>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057d687c0>, 1555092.059882348)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153102861690>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531072f67d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153107017cd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057d68fa0>, 1555094.801135988)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153106d58710>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fcfc0590>, 1555094.407196566)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153107017d50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153106d98b10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fcfc33f0>, 1555095.144512294)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153105ce4490>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153102be5c90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153106ccd710>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153106ab4a90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153107014b10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531058a4c90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153106d97010>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057d6add0>, 1555092.08622992)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153106d73490>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057d6b1c0>, 1555093.617979455)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153106b68310>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057d68bb0>, 1555092.181199502)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153106d59b90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531077df9d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153106b24d50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fcfc37e0>, 1555093.924619289)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153106b256d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302ca39c50>, 1555091.055578535)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15310757b590>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153105569ed0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153105864910>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056d6b9a0>, 1555093.7524924)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153102be5210>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302cc73e00>, 1555092.648605927)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153106ab4950>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057d6bd20>, 1555090.894409174)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531072d2bd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057d68c90>, 1555090.796664775)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153107367590>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057d69240>, 1555090.917540618)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153106d95790>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fcfc25f0>, 1555094.519921982)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531077dd3d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153104018fa0>, 1555094.117049127)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153106b254d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056d6a580>, 1555094.075580388)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531056c3f50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153105798f10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fc7cb9a0>, 1555093.478414897)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15310579b810>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153105b29350>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fc7c9c50>, 1555093.726857303)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153105b28550>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153105815910>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fc7ca970>, 1555093.659773162)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153105816950>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153105cf61d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531060141d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15310617cc90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153105246740>, 1555094.566052116)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15310617c790>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153106d05f90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153105246ac0>, 1555096.047028363)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153106038e90>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153106282590>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531052462e0>, 1555095.820183862)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153106282790>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153106663d90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15322d85cbb0>, 1555095.281071549)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153106b6aa10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153106243d10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15310651dc10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153105036040>, 1555095.58648033)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15310638e8d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fc522510>, 1555095.55841515)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15310651d0d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153102640a90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153106681150>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056d69fd0>, 1555095.31553074)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15310620aed0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153106647d50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153105c52b10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153105bd0890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531050365f0>, 1555095.212393886)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153105cf7ed0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153105bee010>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057d6baf0>, 1555094.336194496)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153105fbc790>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303fcd5ef0>, 1555096.960297862)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153106680250>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153105036190>, 1555093.946762067)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153105cc75d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531050379a0>, 1555095.759933279)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153106646ad0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153105035a90>, 1555094.0875933)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153105c53e10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153105037d90>, 1555093.533026392)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153105547910>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153106abb290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153105036430>, 1555096.117009653)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531040ecd10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531065ef510>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153106904810>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fcfc3930>, 1555095.999059548)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153106905810>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15310617ee90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153106b52f90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057d68130>, 1555096.349823483)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15310617d310>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd1db690>, 1555096.291175577)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153106b50e10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531067dc410>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153106710b50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd1d88a0>, 1555096.364650262)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153105f86650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153105037690>, 1555096.510493623)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153105c53310>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531066d75d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fcee0bb0>, 1555096.761798561)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531066d43d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531067c3d10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fcee2e40>, 1555097.258605214)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531067c1d90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153106ad8090>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303d4504b0>, 1555097.343557756)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153106ada890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153106a65690>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153106b38fd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153105246b30>, 1555097.51570434)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153106a67a10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fcee3850>, 1555097.411161699)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153106b3a250>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15310709cdd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153106fcf110>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057d68830>, 1555095.869218005)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531065ef650>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531063d9f90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531065cf690>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fcee1320>, 1555097.819110118)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15310709c410>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15310780b4d0>, 1555095.71886921)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531065ccf50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b856190>, 1555097.810641439)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153106fcdc10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fc7cbcb0>, 1555095.58853235)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531063da290>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153106f3fed0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b855940>, 1555098.022093073)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153106355b10>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153106d7cad0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153105034590>, 1555097.782966906)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153106d7d050>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153106f93ed0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15310761c290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153104019d30>, 1555097.870667592)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153106f92910>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b8570e0>, 1555097.920158458)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15310761f290>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153106c5c5d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531064049f0>, 1555097.823375073)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153106abe6d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153106baa8d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b855390>, 1555098.063753467)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153106ba9b50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15310726cd10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15310737aed0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153105036200>, 1555097.909717706)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15310726fc50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fc7c8050>, 1555098.918729047)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15310737af50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531071db9d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd347770>, 1555098.535163188)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531071d8250>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153106c9b2d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531074a04d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531079b8590>, 1555098.10459443)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153106c9aa90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531075ad510>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bf1cd00>, 1555098.541045654)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531075ae410>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153107530510>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fcee14e0>, 1555098.620353996)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fd2bb7d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fd497f10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fe3de890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd344600>, 1555098.928917756)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531074a0310>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153106d019d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056d69080>, 1555097.627427521)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153106d018d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15322d85c0c0>, 1555099.243086777)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fd495890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bf1dd30>, 1555099.115649237)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fe3df2d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15310783b910>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bf1ca60>, 1555099.169488905)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153107839b90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531074f4b90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057ea72a0>, 1555099.457536016)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531074f4890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153107745ad0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bf1e9e0>, 1555100.116245799)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531078184d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153107570a90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531052452b0>, 1555099.631384974)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153107573b10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531077cf150>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15310765fc50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd759c50>, 1555099.789620956)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531077c9590>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531077c96d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531017d7590>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd7596a0>, 1555099.640324747)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531077c9990>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd758e50>, 1555099.641492978)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531017d6250>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153057aedf50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd759860>, 1555099.333190892)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153057aec890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153107591bd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fc09dd90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd75bbd0>, 1555100.018680912)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fd1f0510>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd5d9400>, 1555099.444536983)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153057afe510>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531074806d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fcb27e50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302c83d400>, 1555099.738818432)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153107481610>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302c83cc90>, 1555099.879866252)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fcb26c50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531074b9e10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fc12e710>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056c508a0>, 1555100.306606399)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153107531090>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056c533f0>, 1555100.187812299)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fc12dd10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15305794b910>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd759f60>, 1555099.979339949)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531077cd1d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056c52ba0>, 1555100.512140017)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fd24e5d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531051c4450>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056c50c20>, 1555101.132656385)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531051c4a90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153057940e10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fe39cec0>, 1555101.684505351)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153057943010>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fc389890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fc4bef50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bf1c2f0>, 1555100.977570962)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fc3e6bd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153056d0b710>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303f1ef5b0>, 1555101.123858133)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153056d0a410>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fcd8aa50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bf1fbd0>, 1555100.7170148)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fcd8a9d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fc662350>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fe55f1d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd75b4d0>, 1555101.215946656)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fd0e9010>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fc4e97d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056b04bb0>, 1555101.823377164)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fcc63d50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fc5ce850>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fd6c0710>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bf1eeb0>, 1555101.491716031)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fc5cfe10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056b06c10>, 1555101.336360412)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fd6c2a10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530438c7350>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056b04de0>, 1555101.553199452)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530438c5b10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fd414790>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303e165b90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bf1ee40>, 1555101.77791234)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fd415250>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056b07070>, 1555102.262856542)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303e164c10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fd3afd10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b854590>, 1555100.287343333)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530579d1710>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056b06d60>, 1555101.469211436)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153105221210>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056b05390>, 1555102.349807595)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fd3ad6d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fce43c50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057c0ef90>, 1555101.975015546)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303e23e310>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531070ee8d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153042c5e6d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057f08750>, 1555102.005282636)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531070ee650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b855630>, 1555102.468674039)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153042c5fc10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303df8bb90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057f0af20>, 1555102.257186744)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303df8a050>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153042c89510>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fe39d320>, 1555102.441647513)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153042c8a210>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303e137550>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd4e0590>, 1555103.009045116)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303e135a50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303e4a2a50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056b04e50>, 1555103.070165299)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15305566be90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303e33ce10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fc8ccfd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd4e20b0>, 1555103.07406915)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fc8cc410>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fcdc3a90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303dbf5e90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bf1eb30>, 1555103.339826582)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fcdc3d90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd506b30>, 1555104.076145704)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303dbf7510>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303de18c90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd506190>, 1555103.389644322)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303e2c47d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fc4be050>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303de8b4d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056e7a350>, 1555103.578054021)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fd2a5310>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057c0cb40>, 1555103.086583918)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303e275890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303dc73e10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fc327250>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd4e2040>, 1555102.596362383)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531067c2c50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057f0af90>, 1555101.815356944)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fd3a9dd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fcad55c0>, 1555103.446628019)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303dc71d50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303dd59e50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fcad6350>, 1555103.552322511)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303dd5ab10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303db7dbd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fcad60b0>, 1555103.517777582)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303de37790>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303dc5a9d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fe39ef20>, 1555103.891999776)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303dc59a50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303e1d0fd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fcad4c90>, 1555103.852232011)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fc9c8510>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153056c04850>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303db99050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fcad6f20>, 1555104.301022777)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153056c05210>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057fdf700>, 1555103.901099655)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303db99d50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303da7aa50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057fdc520>, 1555103.789770003)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303da7b2d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303e33e9d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303d854690>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057fde660>, 1555104.439719242)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303e33e590>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057fdd320>, 1555104.196419736)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303d8549d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303dabca50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057fdd2b0>, 1555104.546851127)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303dabe010>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303d9c9590>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304284bcb0>, 1555104.833278255)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303d9c9c50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303d82eed0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153042849240>, 1555106.062849819)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303d82d3d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303e1a4750>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303d7111d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303ddc8ed0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530428487c0>, 1555105.002531485)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303d57a710>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304284b460>, 1555104.722079123)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303d710b90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056b06190>, 1555103.13578101)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303ddc8850>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303d8aa950>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153042849e10>, 1555104.858209383)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303d7fe6d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303d5f82d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303d775750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bf1e580>, 1555105.126052544)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303d7fd4d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153042848600>, 1555105.417765383)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303d777810>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303de6e490>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303d62fa90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304284a970>, 1555105.564521559)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303de6e1d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fcad4520>, 1555105.351092253)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303d62f050>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303d572590>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fca3e4a0>, 1555105.074908482)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530552f4f50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303d208710>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fca3cfa0>, 1555105.647071261)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303d2098d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303d462a50>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153102750de0>, 1555105.618466739)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303d462cd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303d46d650>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303d5597d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056b058d0>, 1555105.722012858)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303d46ef90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fcc7ef20>, 1555105.81857054)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303d559bd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303d5bc410>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fcc7cec0>, 1555105.606658828)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303d5bf890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153055f98950>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fcc7cc90>, 1555106.268568548)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153055f9a950>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303d3a77d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fcc7f770>, 1555105.685011789)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303dad9390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303cf8fb10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303d3e7290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fcc7c590>, 1555106.324050087)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303d3e5310>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303d3ca910>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fcc7f070>, 1555106.187816215)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303d222290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fcc7d5c0>, 1555106.121943917)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303d3bda90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303d6f7f90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057fdea50>, 1555106.492543733)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303d993b50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303d0ff150>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303d111e10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fca3e0b0>, 1555106.504770249)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303d1110d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303cf51610>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153042849550>, 1555107.187998076)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153055b2e6d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303d1cd3d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303cefaa10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fce77770>, 1555107.073073305)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153055030b10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fe39e4a0>, 1555106.864988338)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303cefb710>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303cef8050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fca3e120>, 1555107.30105112)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303d9a8ad0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153056150ad0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056c53bd0>, 1555107.358896921)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530561520d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303fd5f810>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fce772a0>, 1555107.732416472)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303fd5eb10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153055aae090>\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303e673f90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fce764a0>, 1555107.521582896)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153055aac790>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fce779a0>, 1555107.728021846)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303e672510>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530560f8490>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153055a4dad0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fce75710>, 1555107.380746078)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530560f80d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fce76120>, 1555107.599354117)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153055a4cb90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303e464cd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fcc25c50>, 1555107.533468641)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303e467390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303e211ed0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fcc7fee0>, 1555107.047076467)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303fc0e6d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fcc24bb0>, 1555107.856047145)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153055e74d90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153055867e10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153055f6ffd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fcc27930>, 1555108.075029137)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153055f6f3d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303f439750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fcc25d30>, 1555108.936541926)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303f1cac90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303d80cd10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303f088b50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fcc260b0>, 1555108.657473044)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303d229c10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fcc24280>, 1555108.233298642)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153034c06a50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303d222a50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153042849be0>, 1555108.306631111)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303cf5ba90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530494dc350>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fc9b56a0>, 1555108.754263926)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153049135cd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153049797410>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fcc7c210>, 1555109.21519713)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153042aafd90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15305515ad90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fc9b5e10>, 1555108.880556498)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153055ce2250>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153055e42ed0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fc9b5550>, 1555110.278971177)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304b685890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153055e9d010>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fc9b4830>, 1555110.615797313)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153055e9f090>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303f0de5d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303d2e6110>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304284bee0>, 1555109.333919937)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303f1aced0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fc9b6740>, 1555108.992940533)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303d2e4350>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153035293a90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fcc24d70>, 1555108.876040799)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153055864890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303d889b10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fcc27230>, 1555107.616004057)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303d88a7d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fc9b6b30>, 1555110.375385162)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153035290690>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fdbfa890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd18f930>, 1555110.542043701)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fdbf8cd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153034d495d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530352066d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fc9b5860>, 1555109.624973834)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153055f6c650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd18c130>, 1555110.13895381)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153035207cd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530549b3810>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd18fb60>, 1555110.060619852)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530549b2790>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153042bce450>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fcad50f0>, 1555110.907444805)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303f254290>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153055c304d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153106cd6120>, 1555110.454544169)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153055cee010>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303547ebd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153106cd6eb0>, 1555110.916952829)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303547ccd0>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153034ff4390>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fce77af0>, 1555110.552519898)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303569d590>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153034d12310>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530351e3010>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153106cd5400>, 1555111.268814775)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530355d4790>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fcad6eb0>, 1555112.033210505)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304984ce90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15305590ff90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153055102190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304306d7d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056b06120>, 1555110.614605895)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303d46d4d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153055cd3f10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fc9b59b0>, 1555110.204315989)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153055103110>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fce76cf0>, 1555110.816398717)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153034d67750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd18ce50>, 1555112.048191374)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303fdd18d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153035837b50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153106cd7f50>, 1555112.306850538)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303efaae90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153055d5df90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fc9b40c0>, 1555111.944300694)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15305549ed50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153034e11a10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153106cd78c0>, 1555111.554006616)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153034e11050>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153034c630d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302c6fa2e0>, 1555112.145714297)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303530b990>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153034cada10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302c6fb930>, 1555112.211155646)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153034caeb90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153034cb0490>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530340fff90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302c6fa580>, 1555111.993647684)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153034cb1510>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd18faf0>, 1555111.92543644)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530340fef10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153054e9d390>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd18dd30>, 1555112.55465224)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153049abf750>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153034144bd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fcc7e7b0>, 1555112.498140355)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153034147550>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303d5de6d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153034c19990>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056b07d20>, 1555113.083213986)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153034cc51d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302c6f9940>, 1555112.631362512)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303fa90050>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303e1466d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303d6b98d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15305503f350>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530353797d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153106cd6c80>, 1555112.473270518)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15305503ca90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd18cd70>, 1555112.617433288)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303d6baad0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fce77700>, 1555112.023639565)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153042a75ad0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302c6f8ad0>, 1555113.877217316)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fda25890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153035379690>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302c6f8f30>, 1555113.163619953)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303e5ffb10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303dd0eb50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b9fe660>, 1555113.273978545)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303dd0c090>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303de78b90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fc9b4600>, 1555114.300543759)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303de7b990>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302db065d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303dfac490>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b9ffaf0>, 1555113.757595698)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304b80ae50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b9ff230>, 1555113.97147001)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303dfad410>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303de552d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b9fe430>, 1555113.684292482)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303de57790>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303d4297d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303d7dad10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302c6fae40>, 1555114.266645323)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304b280f10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b9feb30>, 1555113.966401041)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303d7d8110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153034c8b310>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153034185810>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd18f5b0>, 1555114.264099801)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153034186f50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153034178d90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302c6f8670>, 1555114.594676157)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303417a590>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303d549dd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd18df60>, 1555114.960800568)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303d548910>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302c853010>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302c70e810>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303d69f490>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b9ff310>, 1555114.057599612)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153034c88490>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530555c48d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302c6fb380>, 1555112.853501341)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303d69ea50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd18d470>, 1555112.724660275)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153048d9a810>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bcb11d0>, 1555114.432807539)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302c853450>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bcb2190>, 1555114.577957698)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302c70e450>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153034cddfd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530341053d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bcb2c10>, 1555114.260446098)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153034cdc390>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bcb2510>, 1555114.632847118)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153034106390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153055977590>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bcb0c90>, 1555115.287115322)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302c87e050>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302db4e8d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530554b3ed0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b9fc050>, 1555114.799529592)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302db4cb50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302c6fb8c0>, 1555115.123155809)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530554b0310>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302c9b4ad0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302cade270>, 1555115.380494193)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302c9b4bd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303d5f7510>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302db02b50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302cadd710>, 1555115.468760844)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303df1ea90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302caded60>, 1555115.662067)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302db02e50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302c8e6c50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302cadeeb0>, 1555115.640547865)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302c8e7f10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303de52d50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd18fcb0>, 1555115.57380468)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303de50890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153034c1b790>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b9fc280>, 1555115.942607716)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153034c1b8d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302db00610>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153035598c50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302c8b31d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302c81cb50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bcb08a0>, 1555114.495330977)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302c81c850>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153034cac710>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd18ef90>, 1555115.878839422)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302c8b83d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bcb1c50>, 1555114.436546164)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153034cac6d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153107334f30>, 1555115.848791806)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153034176ad0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153107334830>, 1555116.263119202)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302c8b2b90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302c8a59d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531073346e0>, 1555116.123622045)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302c8a5ad0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302c8863d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b9fff50>, 1555116.159792073)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302c886190>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302c8ac3d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303e072450>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303d357150>, 1555116.458182751)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302c8ac0d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303d3574d0>, 1555116.345486704)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303e073950>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303d1d6110>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303d356d60>, 1555116.483559016)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303d1d50d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303e2a1d50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302c85a9d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303d356190>, 1555116.462049179)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303e2a2610>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303d355320>, 1555116.493601177)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302c85ae90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303dc395d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303d357230>, 1555116.673911443)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303dc3ab10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302c7fcb10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303d354d00>, 1555116.756435547)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302c7ff4d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302c7fd750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303d354520>, 1555117.0692668)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302c7fd690>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302c957450>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303d3549f0>, 1555117.006050301)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302c957150>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302c97cb90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302c91f410>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303d038e50>, 1555117.021914394)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302c97c0d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303d03bee0>, 1555116.850008781)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302c91f390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302c963010>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303d039cc0>, 1555117.281787063)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302c8fa590>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302c8f5ed0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302cae1710>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303d03b380>, 1555118.652241679)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302c8f7b10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302c6f88a0>, 1555118.565743878)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302cae0a90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302cab3c10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303d03b3f0>, 1555118.031655172)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303d5487d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302cab2b10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303d03a580>, 1555117.784103967)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302cab3350>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302ca83050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303d03a510>, 1555117.49693037)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302ca826d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302ca4bc10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303d038280>, 1555118.026738212)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302ca48ed0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302ca37310>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303d0387c0>, 1555117.850963219)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302ca37410>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302c9d3810>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302cbd4b90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303d1430e0>, 1555118.336966855)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302c98fc10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303d356890>, 1555117.721225399)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530358dee90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302cab83d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153107337cb0>, 1555118.363256511)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302cabae10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302cbd2210>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302cbe5950>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303d03af90>, 1555118.591263095)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302c998950>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303d141940>, 1555151.702041626)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302cbe5e50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302c7fc350>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303d141710>, 1555117.853993506)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302c7fdc10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302cb64190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303d140de0>, 1555118.266141864)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302c9154d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153034107bd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303d1416a0>, 1555151.71308119)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302c957690>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302cb7ef10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302cb28550>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bcb2a50>, 1555150.919690883)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302cb7c390>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303d143e00>, 1555118.436981114)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302cb283d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302cb523d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303d03ba10>, 1555151.085227334)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302cb52050>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302cc9dd90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302cade7b0>, 1555151.834508427)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302cbdef10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302ca3dc50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048b24c90>, 1555152.389580779)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302c952190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302c8a5e50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302cc41310>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048b25010>, 1555152.192859514)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302cb29950>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048b25860>, 1555151.042780176)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302cc40cd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302cc62b10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048b279a0>, 1555152.56606005)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302cc63650>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302cc5e910>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302cc170d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048b27620>, 1555151.775096432)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302cab6fd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303d142cf0>, 1555151.687526958)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302cc17dd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302cc2a190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d3c3a50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303d142820>, 1555152.662780923)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302cc2b290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b558ec0>, 1555152.249622867)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d3c2a50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302cc0e890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d3a7d90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b55ad60>, 1555150.944063225)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302cc0e990>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d375b50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302ccc8710>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302ccc5710>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303d0399b0>, 1555150.880758337)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d3a6790>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048b264a0>, 1555152.417935888)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302ccc7a90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b9fd470>, 1555152.65581373)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d375a90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048b25cc0>, 1555118.73139195)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302ccc8e10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d489a90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153107562740>, 1555153.6470164)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153056140f50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d46dd50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153107560520>, 1555153.756967629)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d46ff50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d5958d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302cb7f0d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d368a50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302cc8ea10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b55a970>, 1555152.57672073)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302cb9e8d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303d354280>, 1555152.79966768)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302cc8dc50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d48fd50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d440710>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302cb7c290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303d1439a0>, 1555155.932141535)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531077c4390>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b559fd0>, 1555151.788427365)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d368a90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048974de0>, 1555153.976674371)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d48da10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303d3548a0>, 1555153.811496644)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d441350>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153107563e00>, 1555154.588954352)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d5eb690>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d482150>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303d140130>, 1555155.872053563)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d477750>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d536590>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153103c02dd0>, 1555154.102625005)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d5356d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d5d7050>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d505810>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153107561f60>, 1555154.074986221)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d56c5d0>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303d03b000>, 1555154.113763685)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d504c50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d4613d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d6d5950>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153103c003d0>, 1555155.279692834)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d462a50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048975a90>, 1555155.37150816)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d6d5a90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d6ae650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531043d3850>, 1555156.253231563)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d6ad250>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d66a110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d4b43d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d301a50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302caf44d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d387490>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d543950>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d5204d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d545a90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303d03a270>, 1555153.734067339)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d300690>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153107616b30>, 1555155.641284739)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d6584d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303d142900>, 1555156.086570241)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d4b61d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303d0386e0>, 1555155.136583372)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302c879210>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153103c03380>, 1555154.465763771)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d3851d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153103c014e0>, 1555153.916328068)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d540fd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153103c00c90>, 1555155.82629088)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d51ee10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153107562e40>, 1555155.267484167)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d547150>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d31bed0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531043d1be0>, 1555156.384613156)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d31be90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d681090>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048b26820>, 1555155.850022127)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d681510>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d697cd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531043d1a20>, 1555156.131038658)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d61c4d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d5f7d90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302db8ab10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530488b8440>, 1555156.169489636)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d5f6310>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530488b9400>, 1555157.01258129)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302db8a6d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d760c90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530488baba0>, 1555156.185852506)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d763990>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d78f4d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530488b95c0>, 1555156.32408315)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d78c1d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d731d90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530488bb620>, 1555157.287769837)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d732590>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d743090>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d6fe250>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530488bb000>, 1555157.003802415)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d742190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530488b9630>, 1555156.85720549)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d6fd7d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d8e5b90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530488ba040>, 1555157.365201178)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d8e5d90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d5b8a10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d7a7e50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d7b0990>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531043d23c0>, 1555156.80010431)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d7a48d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153103c01630>, 1555155.722403849)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d4d19d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fc9b41a0>, 1555155.771515907)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d7b1150>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d8b8d90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302cab05d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531043d2510>, 1555157.410054908)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d6cc410>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531043d2eb0>, 1555158.233048784)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302cab3710>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d66a410>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048977cb0>, 1555157.474354049)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d66ae90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d8bc490>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303fce7930>, 1555157.71218841)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d8a7b10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530341ae110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d86a090>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303fce7540>, 1555157.879222801)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d576dd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303fce6200>, 1555157.592600142)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d86ab90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d845350>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303fce7070>, 1555157.939877229)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d846c50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153034c49d90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048b24210>, 1555157.998985201)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153034c4ac90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530341e40d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d965290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303fce79a0>, 1555158.266900128)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530341e4d10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153104f15400>, 1555158.366362339)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d964650>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d8900d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153034127ed0>\n",
      "Evaluating workflow:  58%|█████▊    | 1745/3000 [06:15<02:26,  8.59it/s]Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153104f16c10>, 1555158.170615062)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d893490>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530489750f0>, 1555157.75064216)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d830550>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d6c5f90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302ca3ced0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153104f15b70>, 1555158.72048688)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d83f150>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153104f14de0>, 1555158.337676487)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d978dd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d6423d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302da7d790>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530488bacf0>, 1555158.93545522)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d741210>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153104f14a60>, 1555159.083642751)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302da7cd50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302db3b850>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d7f7a10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153104f157f0>, 1555158.945904453)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302db15890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153104f17770>, 1555158.799665275)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d7f2190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d771ed0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530488b9160>, 1555159.204515222)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d8b8350>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d85d590>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302dbd6c90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530488ba890>, 1555159.324447369)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d8a7e50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153103d04210>, 1555159.371048628)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302dbd6c50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302da89ed0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b559470>, 1555159.309433107)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d92ee50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530341af890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d8f9390>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531044e8980>, 1555159.606748676)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530341ad910>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531044e9b70>, 1555159.150052947)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303410dc50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d99ad50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531044eb230>, 1555159.238174934)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d99a0d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302da9c1d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b55a120>, 1555160.479189314)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302da9fad0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d84f990>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531044e9470>, 1555160.344076374)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d84fe10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302cc9da10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531044e8280>, 1555159.605609003)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302cc9da50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153106b64410>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153106afc650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303d0398d0>, 1555159.595019159)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153106b64050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531044e8b40>, 1555160.688128306)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153106aff810>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153106afcc10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153102440e50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531044eba80>, 1555160.463046708)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153106affe10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303fce6040>, 1555160.255570514)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153102443750>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153106ae21d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531044ea0b0>, 1555160.271913011)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153106ae0cd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531024960d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530488bb0e0>, 1555161.229197085)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153102496810>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531024b6c50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304be481a0>, 1555160.961066951)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531024b6b10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302da7d310>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304be4b8c0>, 1555160.842812763)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d8be790>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531024e5ed0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304be4a510>, 1555160.650617216)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531024608d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531024bbb10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304be495c0>, 1555160.843987998)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531024bb9d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153102478710>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153101f846d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304be4bbd0>, 1555161.544620431)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d946bd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153101facbd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531044eb1c0>, 1555161.316011037)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d544c10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153101fc93d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153103c02d60>, 1555161.616685783)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153101fc91d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15310251c690>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153102000750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531038d6890>, 1555161.216891276)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15310251c790>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531038d4130>, 1555161.958514622)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153102000a90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153101fc0150>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531038d4670>, 1555161.520833945)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153101fc0c10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d9b1490>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153101f58ad0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531038d7460>, 1555160.83647483)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d7e4710>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304be486e0>, 1555161.6988868)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302c87a190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303fce62e0>, 1555161.941597832)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153101f58750>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153034115650>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531024facd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153104f16c80>, 1555162.641483635)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d348290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531038d6ba0>, 1555162.248559504)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531024fb310>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531024434d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531038d4210>, 1555162.200997508)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153106afdc50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153101e52010>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153101e84d10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530488b9320>, 1555163.224283403)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153101e52350>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531038d4e50>, 1555162.642928376)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153101e84bd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153101ec9050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153107560c90>, 1555162.376410989)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153101ec9510>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153101ec9550>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531044ea510>, 1555162.56196146)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153101eca590>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531024b8090>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153101f03f50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd8d5240>, 1555162.724954483)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153101edc110>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd8d4a60>, 1555162.916259368)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153101f03e50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153101e9dbd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd8d6900>, 1555162.639840211)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153101e9f090>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302dabfd90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd8d5470>, 1555162.891904758)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153101eb8d10>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d55f350>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd8d5b00>, 1555163.160573526)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302da41b90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153101f68550>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153101e64b10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd222740>, 1555163.151111414)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153101c87fd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153101ca5010>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd8d6eb0>, 1555162.984609347)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153101ca4e10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153101ca6f10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153101ce4310>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd8d6dd0>, 1555162.767115673)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153101c5ff90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd2222e0>, 1555163.364695106)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153101ca7250>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd2232a0>, 1555163.7186078)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153101ccba50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153101ceb190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304be49400>, 1555163.605119307)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153101ceb510>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153101d0f710>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd222dd0>, 1555163.739417946)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153101d0f5d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153101d073d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd2218d0>, 1555163.732842034)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153101d07e10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153101c91e90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd222120>, 1555164.731677203)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153101e504d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531052b60d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd223460>, 1555164.094612656)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531052b5e50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531052fd3d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153104f16580>, 1555164.266537065)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d88f050>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153105322010>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b659be0>, 1555164.223898567)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153105321c50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153101f16c50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b65ba10>, 1555164.653788206)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153101f172d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153105347b50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531053717d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531053a5a10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b65ac10>, 1555164.562108222)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15310535ff50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b65af20>, 1555164.333278507)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153105371b10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153105305ed0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b65a7b0>, 1555164.590382696)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153105306590>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153105353f50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531028fa190>, 1555164.438597039)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153105353e50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531073b5a90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531073eb310>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b6595c0>, 1555164.971567149)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531053a5d50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b65bee0>, 1555164.834921742)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531073b5810>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b659c50>, 1555165.135375482)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531073e8ad0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153107412ad0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd8d5a90>, 1555165.370796154)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153107413010>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153107433350>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd6b4210>, 1555166.426324543)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153107433210>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15310745eb50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fccb31c0>, 1555165.256929711)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15310745f010>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15310748ad10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fccb0440>, 1555165.127500215)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15310748b410>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153101d204d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531074ab650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fccb2970>, 1555165.40199022)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153101d206d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fccb2a50>, 1555165.596813367)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531074ab510>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153107478450>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fccb0830>, 1555165.46741137)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531074785d0>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531064d3590>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302ca3b460>, 1555166.245344777)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531064d3390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153106517f10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b65add0>, 1555166.297933041)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153106517d10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531065239d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304be4bc40>, 1555166.660568452)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153106523f50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153105306190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302ca3a350>, 1555166.664746265)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153101ecb7d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15310655d7d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531065854d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b65af90>, 1555165.984709767)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153101c4ce90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302ca3a890>, 1555166.545988339)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153106586bd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531065a5390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531073c8f10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd6b6040>, 1555164.747056542)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531073c9050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302ca38c90>, 1555166.316956089)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531065a5190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153107489b90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531062b4910>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302ca3b620>, 1555166.845871608)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531065a7a50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531062e9050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302caff700>, 1555166.747857215)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531062e8e50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531062eaa10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302cafd8d0>, 1555166.862925576)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531062eae10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153106b232d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15310633fed0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302caff380>, 1555166.578894905)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153107494150>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302cafc210>, 1555167.759157714)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15310633fdd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15310637d0d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302cafd630>, 1555167.043372377)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15310637d450>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531062fe6d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15310639de50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302caffaf0>, 1555167.379721591)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531062fe2d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302cafcf30>, 1555167.328089705)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531052fe390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153106349a90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153106385150>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302cafc2f0>, 1555167.004261653)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153106348750>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531061d5e50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302cc73f50>, 1555167.850610935)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531061d5c50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15310620e910>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302cc71470>, 1555168.066137231)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531074104d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153106224910>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153106254110>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b65b770>, 1555166.835400141)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531062b4710>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531062b7610>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302cafd160>, 1555167.902246858)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153106385890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302cafd5c0>, 1555166.400478323)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531062b7410>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302cc70d00>, 1555168.321541232)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153106224a90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304be497f0>, 1555167.93242046)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153106254750>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153106276f90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302cc70910>, 1555168.294273267)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153106277450>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531061f19d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd223d20>, 1555168.215009086)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153105306290>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153106265790>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302ca3aeb0>, 1555168.857231452)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153106265e50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531062fe090>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302d448980>, 1555168.609287271)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153105bbb490>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153106227a90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153105bf4050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302d4492b0>, 1555168.620159002)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153106227c50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302d449390>, 1555168.236234574)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153105bf43d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531062d76d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153105c2b490>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302cc72cf0>, 1555168.841247545)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531062d7750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302d44b690>, 1555168.759934214)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153105c2b350>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153105c45490>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153105c67090>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd6b5b00>, 1555171.018849477)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153105c66f50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153105c81a50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302d44b2a0>, 1555168.983647298)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153105c81910>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153106391e10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153105bf0890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302d44a6d0>, 1555169.41546541)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153105c88610>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302d449940>, 1555169.309026995)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153105bf0690>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153106254fd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531062e8f90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302d44a9e0>, 1555168.474785501)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153105c4d610>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302d34e190>, 1555169.708973491)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531062d4490>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302cc71550>, 1555168.025469455)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531061e11d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15310633fd50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531054f7dd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531038d50f0>, 1555169.851509205)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15310633f750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302cc72510>, 1555169.93704128)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531054f7cd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531055052d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302cafc360>, 1555169.779420675)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153107468910>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153106385dd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15310556fd90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302d34dcc0>, 1555171.239875322)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153106385110>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302d34e040>, 1555169.960198125)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15310556fc50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15310558f7d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302d34c750>, 1555171.41705017)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15310558f690>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153105515f50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302d34ef20>, 1555169.902068808)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153105515ad0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153102a4c7d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153102a67910>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302d34c830>, 1555172.432899137)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153102a4c750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302d4ca890>, 1555170.390635011)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153102a677d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153102aa0950>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302d449f60>, 1555171.115938723)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153102aa1010>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15310557cb90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302d4ca350>, 1555171.774852201)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531062a8390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15310630bc10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153102ae97d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302d4c8520>, 1555170.658628828)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153102abd450>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302d4cad60>, 1555170.443431183)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153102ae95d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153102a62bd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153102b108d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302d4ca820>, 1555170.340793133)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153102a62590>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302d44a200>, 1555170.533162862)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153102b10f50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fccb2200>, 1555172.802358755)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153102ab3250>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153105514710>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302d34d780>, 1555169.462217417)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531054bb990>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531061d6e50>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302d4c9390>, 1555171.338939477)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531061d6fd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531062e2550>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302d5b7070>, 1555172.65421599)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153102b22d50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153102a21090>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302d5b59b0>, 1555172.536782321)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153102a212d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153102aa2210>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302d527230>, 1555172.122081427)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153102a67c10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153102a37690>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302d4497f0>, 1555172.370200161)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153102a37290>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153102b1d410>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531029bc110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531029ede50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531029ad890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302d4ca3c0>, 1555171.388651293)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153102b1d910>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302d5b4ad0>, 1555172.290171575)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531029bc490>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531038d4590>, 1555173.280091051)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531029ee510>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302d4c8440>, 1555172.645645956)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531029adf90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153105bd6850>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304be4ab30>, 1555173.113155155)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153105bd5d50>\n",
      "Evaluating workflow:  58%|█████▊    | 1749/3000 [06:15<01:53, 11.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  58%|█████▊    | 1751/3000 [06:16<01:54, 10.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  58%|█████▊    | 1753/3000 [06:16<01:46, 11.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  58%|█████▊    | 1755/3000 [06:16<02:18,  8.97it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  59%|█████▊    | 1759/3000 [06:17<02:50,  7.29it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  59%|█████▉    | 1763/3000 [06:17<02:16,  9.06it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  59%|█████▉    | 1767/3000 [06:17<01:41, 12.11it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  59%|█████▉    | 1771/3000 [06:18<01:14, 16.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  59%|█████▉    | 1774/3000 [06:18<01:40, 12.17it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  59%|█████▉    | 1776/3000 [06:18<02:25,  8.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  59%|█████▉    | 1780/3000 [06:19<01:56, 10.47it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  59%|█████▉    | 1783/3000 [06:19<01:43, 11.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  60%|█████▉    | 1789/3000 [06:19<01:23, 14.42it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  60%|█████▉    | 1791/3000 [06:19<01:41, 11.92it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  60%|█████▉    | 1793/3000 [06:20<02:19,  8.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  60%|█████▉    | 1795/3000 [06:20<02:11,  9.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  60%|██████    | 1800/3000 [06:20<01:42, 11.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  60%|██████    | 1802/3000 [06:21<01:59, 10.04it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  60%|██████    | 1807/3000 [06:21<01:43, 11.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  60%|██████    | 1811/3000 [06:22<01:54, 10.38it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  60%|██████    | 1813/3000 [06:22<02:01,  9.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  60%|██████    | 1815/3000 [06:22<02:19,  8.49it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  61%|██████    | 1819/3000 [06:22<02:00,  9.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  61%|██████    | 1824/3000 [06:23<01:22, 14.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  61%|██████    | 1826/3000 [06:23<01:26, 13.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  61%|██████    | 1828/3000 [06:23<01:52, 10.40it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  61%|██████    | 1830/3000 [06:23<02:07,  9.14it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  61%|██████    | 1833/3000 [06:24<02:15,  8.63it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  61%|██████    | 1837/3000 [06:24<01:39, 11.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  61%|██████▏   | 1839/3000 [06:24<01:47, 10.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  61%|██████▏   | 1842/3000 [06:24<01:21, 14.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  62%|██████▏   | 1846/3000 [06:25<02:24,  7.97it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  62%|██████▏   | 1848/3000 [06:26<02:46,  6.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  62%|██████▏   | 1851/3000 [06:26<03:00,  6.37it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  62%|██████▏   | 1853/3000 [06:26<02:49,  6.77it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  62%|██████▏   | 1859/3000 [06:27<01:46, 10.67it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  62%|██████▏   | 1861/3000 [06:27<01:42, 11.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  62%|██████▏   | 1865/3000 [06:27<01:47, 10.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  62%|██████▏   | 1867/3000 [06:28<02:11,  8.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  62%|██████▏   | 1870/3000 [06:28<02:18,  8.17it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  62%|██████▏   | 1872/3000 [06:28<01:55,  9.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  62%|██████▏   | 1874/3000 [06:29<02:11,  8.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  63%|██████▎   | 1877/3000 [06:29<02:07,  8.80it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  63%|██████▎   | 1880/3000 [06:29<01:41, 11.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  63%|██████▎   | 1882/3000 [06:29<01:33, 11.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  63%|██████▎   | 1886/3000 [06:30<01:26, 12.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  63%|██████▎   | 1888/3000 [06:30<02:04,  8.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  63%|██████▎   | 1890/3000 [06:30<02:38,  7.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  63%|██████▎   | 1893/3000 [06:31<02:37,  7.04it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  63%|██████▎   | 1896/3000 [06:31<01:45, 10.48it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  63%|██████▎   | 1898/3000 [06:31<01:48, 10.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  63%|██████▎   | 1902/3000 [07:01<58:25,  3.19s/it]  "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "\u001b[32m2026-01-04 10:51:53.970\u001b[0m | \u001b[31m\u001b[1mERROR   \u001b[0m | \u001b[36mevoagentx.workflow.workflow\u001b[0m:\u001b[36masync_execute\u001b[0m:\u001b[36m104\u001b[0m - \u001b[31m\u001b[1mAn Error occurs when executing the workflow: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 1 second. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  63%|██████▎   | 1904/3000 [07:02<43:09,  2.36s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "\u001b[32m2026-01-04 10:51:54.002\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'question': \"You are a domain expert in functional genomics. For experiments carried out in K562 cells, we perturb PELP1 and examine the expression of CDKN1A. Does perturbing PELP1 lead to a significant change in CDKN1A expression? Reply only with 'Final Answer: Yes' or 'Final Answer: No'.\", 'answer': 'Yes', 'question_new': \"You are a domain expert in functional genomics. For experiments carried out in K562 cells, we perturb PELP1 and examine the expression of CDKN1A. Does perturbing PELP1 lead to a significant change in CDKN1A expression? Reply only with 'Final Answer: Yes' or 'Final Answer: No'.\", '_id': 34821}\n",
      "Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 1 second. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n",
      "\u001b[32m2026-01-04 10:51:54.032\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'question': \"You are a domain expert in functional genomics. For experiments carried out in K562 cells, we perturb KIAA0586 and examine the expression of KIAA0586. Does perturbing KIAA0586 lead to a significant change in KIAA0586 expression? Reply only with 'Final Answer: Yes' or 'Final Answer: No'.\", 'answer': 'Yes', 'question_new': \"You are a domain expert in functional genomics. For experiments carried out in K562 cells, we perturb KIAA0586 and examine the expression of KIAA0586. Does perturbing KIAA0586 lead to a significant change in KIAA0586 expression? Reply only with 'Final Answer: Yes' or 'Final Answer: No'.\", '_id': 22981}\n",
      "Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 1 second. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  64%|██████▎   | 1908/3000 [07:03<23:52,  1.31s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  64%|██████▎   | 1910/3000 [07:03<16:50,  1.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  64%|██████▍   | 1917/3000 [07:04<06:42,  2.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  64%|██████▍   | 1919/3000 [07:04<07:01,  2.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  64%|██████▍   | 1923/3000 [07:05<04:23,  4.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  64%|██████▍   | 1926/3000 [07:05<03:29,  5.11it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  64%|██████▍   | 1931/3000 [07:06<03:08,  5.67it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  64%|██████▍   | 1934/3000 [07:06<02:23,  7.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  65%|██████▍   | 1936/3000 [07:06<02:06,  8.40it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  65%|██████▍   | 1940/3000 [07:07<01:54,  9.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  65%|██████▍   | 1944/3000 [07:07<01:55,  9.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  65%|██████▍   | 1946/3000 [07:08<02:45,  6.35it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  65%|██████▌   | 1950/3000 [07:08<02:07,  8.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  65%|██████▌   | 1953/3000 [07:08<01:39, 10.48it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  65%|██████▌   | 1955/3000 [07:08<01:40, 10.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  65%|██████▌   | 1959/3000 [07:09<02:03,  8.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  65%|██████▌   | 1961/3000 [07:09<02:11,  7.88it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  66%|██████▌   | 1965/3000 [07:10<01:46,  9.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  66%|██████▌   | 1970/3000 [07:10<01:26, 11.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  66%|██████▌   | 1972/3000 [07:11<02:07,  8.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  66%|██████▌   | 1974/3000 [07:11<02:04,  8.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  66%|██████▌   | 1977/3000 [07:11<01:51,  9.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  66%|██████▌   | 1981/3000 [07:11<01:48,  9.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  66%|██████▌   | 1984/3000 [07:12<01:54,  8.84it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  66%|██████▌   | 1986/3000 [07:12<01:50,  9.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  66%|██████▋   | 1990/3000 [07:12<01:14, 13.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  66%|██████▋   | 1992/3000 [07:12<01:28, 11.37it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  67%|██████▋   | 1996/3000 [07:13<01:37, 10.29it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  67%|██████▋   | 1999/3000 [07:13<02:02,  8.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  67%|██████▋   | 2002/3000 [07:13<01:24, 11.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  67%|██████▋   | 2004/3000 [07:14<01:43,  9.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  67%|██████▋   | 2006/3000 [07:14<01:49,  9.07it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  67%|██████▋   | 2010/3000 [07:14<01:31, 10.78it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  67%|██████▋   | 2014/3000 [07:15<01:32, 10.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  67%|██████▋   | 2016/3000 [07:15<02:05,  7.84it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  67%|██████▋   | 2019/3000 [07:16<02:22,  6.89it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  67%|██████▋   | 2022/3000 [07:16<02:11,  7.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  68%|██████▊   | 2029/3000 [07:16<01:02, 15.47it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  68%|██████▊   | 2031/3000 [07:17<02:19,  6.92it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  68%|██████▊   | 2033/3000 [07:17<02:12,  7.28it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  68%|██████▊   | 2035/3000 [07:18<02:12,  7.28it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  68%|██████▊   | 2040/3000 [07:18<01:31, 10.46it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  68%|██████▊   | 2042/3000 [07:18<01:25, 11.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  68%|██████▊   | 2046/3000 [07:18<01:23, 11.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  68%|██████▊   | 2048/3000 [07:19<01:20, 11.82it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  68%|██████▊   | 2050/3000 [07:19<02:04,  7.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  68%|██████▊   | 2055/3000 [07:19<01:34, 10.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  69%|██████▊   | 2057/3000 [07:20<01:47,  8.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  69%|██████▊   | 2061/3000 [07:20<01:21, 11.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  69%|██████▉   | 2063/3000 [07:20<01:13, 12.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  69%|██████▉   | 2066/3000 [07:20<01:13, 12.66it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  69%|██████▉   | 2068/3000 [07:21<01:33,  9.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  69%|██████▉   | 2071/3000 [07:21<01:25, 10.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  69%|██████▉   | 2073/3000 [07:21<01:42,  9.07it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  69%|██████▉   | 2077/3000 [07:22<01:28, 10.46it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  69%|██████▉   | 2079/3000 [07:22<01:22, 11.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  69%|██████▉   | 2083/3000 [07:22<01:12, 12.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  70%|██████▉   | 2085/3000 [07:22<01:15, 12.14it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  70%|██████▉   | 2087/3000 [07:22<01:20, 11.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  70%|██████▉   | 2089/3000 [07:23<01:27, 10.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  70%|██████▉   | 2092/3000 [07:23<01:47,  8.47it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  70%|██████▉   | 2095/3000 [07:23<01:28, 10.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  70%|██████▉   | 2097/3000 [07:23<01:18, 11.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  70%|███████   | 2102/3000 [07:24<00:59, 15.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  70%|███████   | 2106/3000 [07:24<01:09, 12.78it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  70%|███████   | 2108/3000 [07:24<01:12, 12.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  70%|███████   | 2110/3000 [07:25<01:44,  8.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  71%|███████   | 2116/3000 [07:25<01:07, 13.11it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  71%|███████   | 2118/3000 [07:25<01:24, 10.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  71%|███████   | 2120/3000 [07:26<01:57,  7.52it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  71%|███████   | 2127/3000 [07:27<01:41,  8.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  71%|███████   | 2133/3000 [07:27<01:11, 12.06it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  71%|███████   | 2135/3000 [07:27<01:10, 12.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  71%|███████   | 2137/3000 [07:28<02:14,  6.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  71%|███████▏  | 2139/3000 [07:28<02:03,  6.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  71%|███████▏  | 2141/3000 [07:28<02:05,  6.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  72%|███████▏  | 2147/3000 [07:29<01:10, 12.11it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  72%|███████▏  | 2149/3000 [07:29<01:14, 11.48it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  72%|███████▏  | 2151/3000 [07:29<01:31,  9.29it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  72%|███████▏  | 2153/3000 [07:29<01:38,  8.57it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  72%|███████▏  | 2158/3000 [07:30<01:20, 10.48it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  72%|███████▏  | 2160/3000 [07:30<01:37,  8.60it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  72%|███████▏  | 2162/3000 [07:31<01:49,  7.67it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  72%|███████▏  | 2166/3000 [07:31<01:29,  9.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  72%|███████▏  | 2168/3000 [07:31<01:26,  9.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  72%|███████▏  | 2170/3000 [07:31<01:21, 10.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  72%|███████▏  | 2173/3000 [08:01<48:58,  3.55s/it]  "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  73%|███████▎  | 2176/3000 [08:02<28:47,  2.10s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "\u001b[32m2026-01-04 10:52:54.025\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'question': \"You are a functional genomics specialist. In K562 cells, we perturb NUP62 and monitor PRC1 expression. Decide whether this perturbation leads to a significant alteration in PRC1 expression. Answer only in the format 'Final Answer: Yes' or 'Final Answer: No'.\", 'answer': 'Yes', 'question_new': \"You are a functional genomics specialist. In K562 cells, we perturb NUP62 and monitor PRC1 expression. Decide whether this perturbation leads to a significant alteration in PRC1 expression. Answer only in the format 'Final Answer: Yes' or 'Final Answer: No'.\", '_id': 33328}\n",
      "Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 1 second. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n",
      "\u001b[32m2026-01-04 10:52:54.072\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'question': \"You are a domain expert in functional genomics. For experiments carried out in K562 cells, we perturb EIF3M and examine the expression of SNHG7. Does perturbing EIF3M lead to a significant change in SNHG7 expression? Reply only with 'Final Answer: Yes' or 'Final Answer: No'.\", 'answer': 'Yes', 'question_new': \"You are a domain expert in functional genomics. For experiments carried out in K562 cells, we perturb EIF3M and examine the expression of SNHG7. Does perturbing EIF3M lead to a significant change in SNHG7 expression? Reply only with 'Final Answer: Yes' or 'Final Answer: No'.\", '_id': 14591}\n",
      "Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 1 second. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  73%|███████▎  | 2178/3000 [08:02<22:04,  1.61s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2026-01-04 10:52:54.086\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'question': \"You are an expert in gene regulation studies. For experiments performed in K562 cells, ALG1 is perturbed and MANF expression is measured. Determine whether MANF exhibits a significant expression change under this perturbation. Respond exactly with 'Final Answer: Yes' or 'Final Answer: No'.\", 'answer': 'Yes', 'question_new': \"You are an expert in gene regulation studies. For experiments performed in K562 cells, ALG1 is perturbed and MANF expression is measured. Determine whether MANF exhibits a significant expression change under this perturbation. Respond exactly with 'Final Answer: Yes' or 'Final Answer: No'.\", '_id': 1139}\n",
      "Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 1 second. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n",
      "\u001b[32m2026-01-04 10:52:54.102\u001b[0m | \u001b[31m\u001b[1mERROR   \u001b[0m | \u001b[36mevoagentx.workflow.workflow\u001b[0m:\u001b[36masync_execute\u001b[0m:\u001b[36m104\u001b[0m - \u001b[31m\u001b[1mAn Error occurs when executing the workflow: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 1 second. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "\u001b[32m2026-01-04 10:52:54.104\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'question': \"You are an expert analyst of perturbation datasets. For K562 cells subjected to perturbation of BCLAF1, does the expression profile of BCLAF1 indicate a significant change relative to control conditions? Reply strictly with 'Final Answer: Yes' or 'Final Answer: No'.\", 'answer': 'Yes', 'question_new': \"You are an expert analyst of perturbation datasets. For K562 cells subjected to perturbation of BCLAF1, does the expression profile of BCLAF1 indicate a significant change relative to control conditions? Reply strictly with 'Final Answer: Yes' or 'Final Answer: No'.\", '_id': 3457}\n",
      "Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 1 second. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n",
      "\u001b[32m2026-01-04 10:52:54.210\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'question': \"You are an expert in gene regulation studies. For experiments performed in K562 cells, RRM2 is perturbed and CDKN1A expression is measured. Determine whether CDKN1A exhibits a significant expression change under this perturbation. Respond exactly with 'Final Answer: Yes' or 'Final Answer: No'.\", 'answer': 'Yes', 'question_new': \"You are an expert in gene regulation studies. For experiments performed in K562 cells, RRM2 is perturbed and CDKN1A expression is measured. Determine whether CDKN1A exhibits a significant expression change under this perturbation. Respond exactly with 'Final Answer: Yes' or 'Final Answer: No'.\", '_id': 45979}\n",
      "Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 1 second. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  73%|███████▎  | 2182/3000 [08:02<12:21,  1.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2026-01-04 10:52:54.246\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'question': \"You are a domain expert in functional genomics. For experiments carried out in K562 cells, we perturb DHX36 and examine the expression of NBPF10. Does perturbing DHX36 lead to a significant change in NBPF10 expression? Reply only with 'Final Answer: Yes' or 'Final Answer: No'.\", 'answer': 'Yes', 'question_new': \"You are a domain expert in functional genomics. For experiments carried out in K562 cells, we perturb DHX36 and examine the expression of NBPF10. Does perturbing DHX36 lead to a significant change in NBPF10 expression? Reply only with 'Final Answer: Yes' or 'Final Answer: No'.\", '_id': 12201}\n",
      "Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 1 second. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n",
      "\u001b[32m2026-01-04 10:52:54.256\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'question': \"You are a domain expert in functional genomics. For experiments carried out in K562 cells, we perturb UTP20 and examine the expression of KNL1. Does perturbing UTP20 lead to a significant change in KNL1 expression? Reply only with 'Final Answer: Yes' or 'Final Answer: No'.\", 'answer': 'Yes', 'question_new': \"You are a domain expert in functional genomics. For experiments carried out in K562 cells, we perturb UTP20 and examine the expression of KNL1. Does perturbing UTP20 lead to a significant change in KNL1 expression? Reply only with 'Final Answer: Yes' or 'Final Answer: No'.\", '_id': 58561}\n",
      "Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 1 second. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  73%|███████▎  | 2185/3000 [08:03<09:06,  1.49it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  73%|███████▎  | 2187/3000 [08:04<07:55,  1.71it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  73%|███████▎  | 2191/3000 [08:04<04:54,  2.74it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  73%|███████▎  | 2193/3000 [08:05<05:45,  2.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  73%|███████▎  | 2197/3000 [08:05<03:26,  3.88it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  73%|███████▎  | 2203/3000 [08:06<01:55,  6.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  74%|███████▎  | 2206/3000 [08:06<02:11,  6.06it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  74%|███████▎  | 2208/3000 [08:07<02:25,  5.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  74%|███████▎  | 2210/3000 [08:07<02:10,  6.07it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  74%|███████▍  | 2213/3000 [08:07<01:37,  8.04it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  74%|███████▍  | 2217/3000 [08:07<01:29,  8.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  74%|███████▍  | 2219/3000 [08:08<01:17, 10.05it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  74%|███████▍  | 2221/3000 [08:08<01:17, 10.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  74%|███████▍  | 2223/3000 [08:08<01:34,  8.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  74%|███████▍  | 2227/3000 [08:08<01:15, 10.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  74%|███████▍  | 2229/3000 [08:09<01:07, 11.46it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  74%|███████▍  | 2231/3000 [08:09<01:27,  8.83it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  75%|███████▍  | 2236/3000 [08:09<01:02, 12.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  75%|███████▍  | 2238/3000 [08:09<00:59, 12.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  75%|███████▍  | 2242/3000 [08:10<01:25,  8.88it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  75%|███████▍  | 2246/3000 [08:10<01:10, 10.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  75%|███████▍  | 2248/3000 [08:11<01:22,  9.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  75%|███████▌  | 2250/3000 [08:11<01:22,  9.11it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  75%|███████▌  | 2252/3000 [08:11<01:46,  7.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  75%|███████▌  | 2255/3000 [08:12<01:38,  7.60it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  75%|███████▌  | 2261/3000 [08:12<00:54, 13.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  75%|███████▌  | 2263/3000 [08:12<00:57, 12.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  76%|███████▌  | 2267/3000 [08:12<01:05, 11.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  76%|███████▌  | 2271/3000 [08:13<01:26,  8.38it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  76%|███████▌  | 2273/3000 [08:13<01:23,  8.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  76%|███████▌  | 2276/3000 [08:13<01:08, 10.57it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  76%|███████▌  | 2280/3000 [08:14<01:00, 11.86it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  76%|███████▌  | 2282/3000 [08:14<00:55, 12.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  76%|███████▌  | 2284/3000 [08:14<01:04, 11.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  76%|███████▌  | 2286/3000 [08:15<01:25,  8.35it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  76%|███████▋  | 2291/3000 [08:15<01:20,  8.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  76%|███████▋  | 2293/3000 [08:15<01:09, 10.11it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  76%|███████▋  | 2295/3000 [08:16<02:00,  5.87it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  77%|███████▋  | 2299/3000 [08:16<01:31,  7.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  77%|███████▋  | 2303/3000 [08:17<01:14,  9.37it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  77%|███████▋  | 2305/3000 [08:17<01:17,  8.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  77%|███████▋  | 2307/3000 [08:17<01:43,  6.67it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  77%|███████▋  | 2309/3000 [08:18<01:59,  5.78it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  77%|███████▋  | 2316/3000 [08:18<00:53, 12.82it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  77%|███████▋  | 2320/3000 [08:18<00:51, 13.11it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  77%|███████▋  | 2322/3000 [08:19<00:53, 12.71it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  77%|███████▋  | 2324/3000 [08:19<00:59, 11.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  78%|███████▊  | 2326/3000 [08:20<01:45,  6.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  78%|███████▊  | 2330/3000 [08:20<01:19,  8.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  78%|███████▊  | 2335/3000 [08:20<00:46, 14.38it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  78%|███████▊  | 2338/3000 [08:20<00:47, 14.05it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  78%|███████▊  | 2340/3000 [08:20<00:52, 12.60it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  78%|███████▊  | 2343/3000 [08:21<00:52, 12.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  78%|███████▊  | 2347/3000 [08:21<01:09,  9.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  78%|███████▊  | 2351/3000 [08:22<01:00, 10.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  78%|███████▊  | 2353/3000 [08:22<01:05,  9.83it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  78%|███████▊  | 2355/3000 [08:22<01:00, 10.63it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  79%|███████▊  | 2359/3000 [08:22<00:56, 11.42it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  79%|███████▊  | 2361/3000 [08:22<00:52, 12.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  79%|███████▉  | 2365/3000 [08:23<00:54, 11.67it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  79%|███████▉  | 2369/3000 [08:23<00:47, 13.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  79%|███████▉  | 2371/3000 [08:23<00:59, 10.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  79%|███████▉  | 2373/3000 [08:24<01:27,  7.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  79%|███████▉  | 2379/3000 [08:24<00:53, 11.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  79%|███████▉  | 2381/3000 [08:24<00:54, 11.37it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  80%|███████▉  | 2385/3000 [08:25<00:54, 11.38it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  80%|███████▉  | 2387/3000 [08:25<00:53, 11.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  80%|███████▉  | 2389/3000 [08:25<00:57, 10.57it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  80%|███████▉  | 2393/3000 [08:26<01:00, 10.05it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  80%|███████▉  | 2396/3000 [08:26<00:46, 13.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  80%|███████▉  | 2398/3000 [08:26<00:54, 10.97it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  80%|████████  | 2400/3000 [08:27<01:34,  6.37it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  80%|████████  | 2402/3000 [08:27<01:21,  7.36it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  80%|████████  | 2405/3000 [08:27<01:27,  6.77it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  80%|████████  | 2410/3000 [08:27<00:49, 12.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  80%|████████  | 2414/3000 [08:28<00:42, 13.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  81%|████████  | 2416/3000 [08:28<01:26,  6.71it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  81%|████████  | 2418/3000 [08:29<01:23,  7.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  81%|████████  | 2420/3000 [08:29<01:13,  7.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  81%|████████  | 2424/3000 [08:29<01:00,  9.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  81%|████████  | 2426/3000 [08:29<00:54, 10.61it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  81%|████████  | 2428/3000 [08:30<01:54,  4.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  81%|████████  | 2435/3000 [08:30<00:53, 10.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  81%|████████▏ | 2438/3000 [08:31<01:02,  9.04it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  81%|████████▏ | 2442/3000 [08:31<00:57,  9.66it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  81%|████████▏ | 2444/3000 [08:31<00:52, 10.66it/s]Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15310281a1d0>\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302d524910>, 1555173.868770099)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15310281a050>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531027fbfd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302d448b40>, 1555174.250484609)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531027f9410>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153102b22e90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531027fecd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153102785e10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15310263af50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531027d78d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302d525f60>, 1555172.827879)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531027fead0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302d4cb930>, 1555172.865597506)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153102b22e50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302d527d90>, 1555172.975668456)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153102786510>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302d34f620>, 1555172.987922175)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15310263b610>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302d527620>, 1555174.373212221)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531027d76d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15310263bcd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302d525a20>, 1555174.721329571)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15310263bd50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531026ddf90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530ff9de890>, 1555174.660165769)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153102402b90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153102668590>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302d524440>, 1555174.568684482)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531026690d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153102273890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15310389d470>, 1555176.585734914)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15310208dd90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153102347e50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15310389c3d0>, 1555177.162889306)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153101d45050>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153102819610>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303cf47c40>, 1555176.071442739)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153102995b90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15310204f590>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048f26f90>, 1555175.74179687)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15310204f390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153101d9e3d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048c0c360>, 1555177.584781301)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153101d9f310>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531023292d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048c0fd90>, 1555177.363489934)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153101e09e90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153105b141d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048c0e3c0>, 1555179.425129105)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531022b3410>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15310591a890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056ee66d0>, 1555213.023623716)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15310591a910>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15310232a490>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153102113d90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530ffbdc0c0>, 1555179.361751354)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153101d7b3d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530ffbdc2f0>, 1555179.12112748)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153102329490>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531059750d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056e07bd0>, 1555212.616493311)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153105974710>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153105af5810>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530ffbdcf30>, 1555214.3081281)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531026bf610>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531058dd250>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153107204350>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531059304d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531071e3b90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15310720cf90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15310726b1d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048c0c8a0>, 1555212.458277969)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153107273b10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056e06190>, 1555211.397579511)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531058dd910>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153106130690>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056e05390>, 1555211.021793146)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153107205050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15310389c210>, 1555212.828513347)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531071c43d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056e05010>, 1555214.448136059)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531071e3a50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056e077e0>, 1555214.240473323)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15310720dd90>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302d3d9da0>, 1555214.824000029)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153106130490>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153106103f90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153106101810>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302d3daeb0>, 1555215.883104971)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15310610bdd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056ee4ec0>, 1555214.065464291)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153106101bd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531072a8490>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302d3db000>, 1555214.652318841)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531072ab010>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531061a3250>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302c995160>, 1555215.393839484)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531061a3050>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531071c6ad0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302c996ac0>, 1555214.348194951)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15310614a090>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531060f50d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302cb8f2a0>, 1555215.532388142)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531060f51d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153107192310>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302c9944b0>, 1555214.846931624)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531060f5890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153107102990>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302c995400>, 1555216.252589128)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531071c9f10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153107709610>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302cb8fc40>, 1555216.1110512)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531077094d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15310712d410>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302cb8d0f0>, 1555215.720564059)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15310712db50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153107718e90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153107728050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302cb8ce50>, 1555215.628446984)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531077196d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530ffbdc280>, 1555215.942608978)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153107728590>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153107758f90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056ee5e10>, 1555215.915632808)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153107759bd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153107795d10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531076cd630>, 1555217.424153138)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153107795bd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531076bd950>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531076cee40>, 1555216.812248792)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531076be6d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15310619a290>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153102c6b750>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153102c84250>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302d3db230>, 1555215.466695616)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153107735890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531076ce740>, 1555218.409619933)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153102c6bbd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531076ce970>, 1555218.390905077)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153102c848d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15310698b4d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153105ee4670>, 1555220.183997217)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153105ef2090>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153105f1c5d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153105ee42f0>, 1555219.393318796)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153105f1c810>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153102cd7590>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302cb8fcb0>, 1555217.587259645)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153102cd6690>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153105f6e010>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153105ee5c50>, 1555219.481251725)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153105f6df10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153102c7b190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153105ee6ba0>, 1555219.761022168)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153102c7bb10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153105e3c810>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153105e34750>, 1555220.431750366)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153105e3c790>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153105f81650>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153105f7d810>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531076cf700>, 1555219.47554743)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531071595d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153105ee6c80>, 1555219.560033459)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153105f7d910>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153106eb73d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153106c892b0>, 1555221.726086864)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153106eb7690>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531068d45d0>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153105e349f0>, 1555222.11509113)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153106f13610>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153106db7450>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153106bdd850>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153105e37e70>, 1555220.961811151)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153106bddb50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153106dc4590>, 1555222.360385892)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153106db7650>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153106e03310>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153106dc5240>, 1555223.033247182)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153106e02dd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153042c17c10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531076cd160>, 1555224.006022709)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153042c17a90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153106decc10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153106dc4c90>, 1555225.278190348)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153106dece90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153042cf1f50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153106def510>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153042c490f0>, 1555225.531228781)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153042c6e350>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153106dc5010>, 1555222.332125966)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153106def3d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530435c1a50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530435e0810>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153042c4b3f0>, 1555225.16135559)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530435c1910>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153042c4bcb0>, 1555225.189895251)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530435da190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153043500350>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153042c49550>, 1555225.327531579)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153043500a10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530435c0410>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153043510830>, 1555224.834698209)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530435c0b90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153043537010>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153105ee5a90>, 1555225.469624424)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153043535890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153043528d50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153043510360>, 1555225.146952784)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153043529610>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153043582290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153043511320>, 1555225.123027916)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153043581210>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153043553790>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530435125f0>, 1555225.598070195)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153106e53f50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153042cf8550>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304313d510>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153043512ac0>, 1555225.82309075)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153042cf9390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153043147e50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153106dc7540>, 1555226.083252684)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153043147d10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153043144110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153106f5ffd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153042cf9790>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153043532450>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153043513380>, 1555225.989549014)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304313d350>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153106dc6740>, 1555226.659229503)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153042c8dc10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153042c4aac0>, 1555223.928282604)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153042cf2b90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153106c8a040>, 1555223.911358603)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153042cf9990>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153042c4a900>, 1555224.821111663)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153043531210>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153043190810>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531027954d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531027a7ed0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531027d2350>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153102645590>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531026bd350>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531027ea290>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531026ea4d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153102722890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531026d1290>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153102345090>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15310237f550>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531023b2190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153102350f90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531023ca650>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153102400cd0>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531027f86d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531027fab50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15310229d2d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153102260490>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153102374a10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531022635d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531022b3750>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15310230d910>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153102329390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531027ebb90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531020bad10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531020d9190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531020ff490>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531023756d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15310206ff10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153102112e10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153101d76010>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531022eb4d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153102083590>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153101dc8910>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153101dcab10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153101e08410>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153101dd3150>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531067ed9d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531067f6850>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153101e2e6d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153106851f50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15310686f950>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531067b8810>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153101d76cd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153101df4590>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153106830e10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153105abcd50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153105ad7410>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153105b2c310>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153105b4b2d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153105b6b710>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153105ba6710>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153102654690>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531058e0d50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15310680f610>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153105954090>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531058c2f50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15310594c6d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153105998f10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15310715b090>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153107181a50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531070d1b50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531060ed3d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153107791dd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531061253d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153102c37f50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15310722f990>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153102ce1a50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531060f4b50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153102d13ed0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153106180a10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153102c7b350>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153107758350>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531068e1010>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531068fbf90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153102c4cbd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153106930590>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153107797990>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153106979510>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531069a54d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531069321d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153105ebfad0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153105ec5c50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153105f4d550>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153105f12650>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153105dfb210>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153106979450>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153105e32910>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153105e5c690>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153105e8d4d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153105e0a250>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153105eaed10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153106bf6550>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153105faef90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153106bbf3d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153105f4eb10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153106c3d990>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153105f106d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153106c82f50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153106c15650>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153106ede390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153106ecd3d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153106f20510>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153106f3a990>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153106f38ed0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153106f97f10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153106db5450>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153106dde6d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153106dd6950>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153106e44850>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153106ea0ed0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153106e504d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153042c2ec50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153042c17050>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153042c8d2d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153042cbb650>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153106e41a10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153042c648d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153042c4a200>, 1555224.833612513)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153042c65c10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302d3d8de0>, 1555224.826914712)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153106dd6910>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153106dc7e70>, 1555224.83059377)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153042c17110>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153043194360>, 1555227.270437144)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153043190690>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302d5258d0>, 1555172.250425047)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153102796bd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302d524d70>, 1555172.768042939)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531027a5890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302d527af0>, 1555172.162806795)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531027d2150>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302d5b4d00>, 1555173.890109514)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531026458d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048f27bd0>, 1555173.64062)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531026bd150>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048f24c20>, 1555174.133052845)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531027ea210>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048f24910>, 1555173.738845595)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153105be2a50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048f27d20>, 1555174.258656377)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153102722690>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302d448280>, 1555174.108575926)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531026d1090>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302d34ecf0>, 1555174.266981666)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153102345c90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302d5271c0>, 1555174.611912292)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15310237f3d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302d526f20>, 1555174.650693131)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531023b2050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302cafe890>, 1555174.89007523)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153102351610>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530ff9dd320>, 1555174.959406107)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531023ca450>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530ff9dcc20>, 1555175.040890178)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153102400ad0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302d525e10>, 1555175.172433503)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153102377f50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530ff9df000>, 1555175.946487525)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15310277c410>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302d5b57f0>, 1555176.099537875)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15310229da10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303cf454e0>, 1555176.000138333)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531026de6d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530ff9dd550>, 1555175.22326268)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531023748d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302d449b70>, 1555175.350172229)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531022633d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303cf45160>, 1555175.719703241)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531022b3610>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303cf455c0>, 1555175.835014893)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15310230d510>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302d5274d0>, 1555176.291467788)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153102329a90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303cf46c10>, 1555176.770469729)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531027ebad0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15310389c280>, 1555176.436532659)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531020bab10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15310389f070>, 1555176.806875615)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531020d8e10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302d44ac10>, 1555177.043964888)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531020ff290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15310389c750>, 1555176.471770246)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153102375ad0>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15310389db70>, 1555176.964736177)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15310206fe10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15310389e200>, 1555177.427138921)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153102113450>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302d527690>, 1555177.116923007)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153101d764d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303cf44de0>, 1555176.51771812)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531022eb2d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302d5256a0>, 1555176.269546805)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153102083390>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303cf47310>, 1555177.275942749)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153101dc87d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048c0ecf0>, 1555177.620597065)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153101dca0d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048c0d4e0>, 1555178.164776247)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153101df66d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048c0fd20>, 1555178.053131024)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153101dd3010>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048c0faf0>, 1555178.251880784)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531067edd50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048c0fe00>, 1555178.075285192)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531067f6650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15310389d940>, 1555178.754824811)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531022e80d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303cf47460>, 1555178.5564106)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531068526d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303cf45550>, 1555179.141033134)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15310686f810>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15310389d550>, 1555179.313372635)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531067b8ed0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530ffbdc130>, 1555178.963174825)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531067ba090>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302d525010>, 1555178.275383728)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153101df4390>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048c0de80>, 1555178.663307463)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531068312d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15310389f2a0>, 1555179.303789417)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153105abccd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530ffbdf5b0>, 1555179.590557921)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153105ad72d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530ffbdd390>, 1555212.52397397)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153105b2c810>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530ffbdeeb0>, 1555212.520500747)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153105b4b0d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530ffbdc750>, 1555212.705329184)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153105b6b510>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056ee5ef0>, 1555212.830415765)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153105ba6e10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056ee62e0>, 1555213.084126288)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153101d8e650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056ee5010>, 1555211.189714957)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531067e8e50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056ee69e0>, 1555211.178495138)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15310680f310>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056ee6b30>, 1555212.628717993)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531059543d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056ee5630>, 1555211.77688796)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531058c3090>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530ffbddef0>, 1555212.371685274)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15310594c4d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056e05fd0>, 1555211.285699747)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153105996410>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302cb8d470>, 1555214.631993924)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15310715af10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302cb8e040>, 1555214.911673211)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153107181850>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302cb8db70>, 1555214.661301911)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531070d2550>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056ee57f0>, 1555216.57313282)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531060ec5d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531076ccde0>, 1555216.397216617)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531077924d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531076cd240>, 1555216.634294672)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153106126450>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531076cdef0>, 1555216.421623087)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153102c37e50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531076ce0b0>, 1555216.94623346)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15310722f750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531076cc2f0>, 1555217.202435741)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153102ce1dd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531076cf3f0>, 1555217.457084544)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153106102110>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153102c482f0>, 1555220.937155416)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153102d13d90>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302c9940c0>, 1555216.746520892)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153107298050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302d3da2e0>, 1555217.754906351)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153102c78b10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302c9963c0>, 1555217.906646413)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153107759d10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153102c49390>, 1555218.364992439)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531068e16d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153102c495c0>, 1555219.322369187)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531068fbe90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153102c49be0>, 1555217.902824964)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153107102f10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302cb8f770>, 1555218.303154059)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153106930450>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153102c4a970>, 1555218.499172869)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15310691f6d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302d3d9cc0>, 1555218.399143138)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153106979310>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531076cc910>, 1555218.598209337)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531069a4d90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056e04600>, 1555218.65421946)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531069333d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056ee7cb0>, 1555218.610503651)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153105ebf990>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153102c4bf50>, 1555219.156993313)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153105ec7850>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303cf45400>, 1555219.551926492)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153105f4d350>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153105ee5470>, 1555219.222113512)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153105f13550>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302cb8cbb0>, 1555220.85355155)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153105dfab90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153102c4add0>, 1555220.506033332)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15310697af90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153105e34520>, 1555220.612679733)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153105e327d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153105e34980>, 1555220.339819404)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153105e5cd50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153105e351d0>, 1555220.763643704)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153105e8d410>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153102c49f60>, 1555220.008636261)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153105e0a410>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153105e359b0>, 1555220.708075244)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153105eae950>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153105e36ba0>, 1555221.971721262)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153106bf6410>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302cb8cfa0>, 1555220.073552025)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153105fae6d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153105e36f90>, 1555221.149507555)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153106bbf790>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153102c491d0>, 1555221.328838353)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153105f4e3d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056e070e0>, 1555221.692189959)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153106c3d790>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153106c88360>, 1555221.545284079)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153106c63f50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153105ee5b00>, 1555221.491689275)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153106c83690>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153105ee4280>, 1555221.550712744)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153106c15550>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153106c897f0>, 1555222.328264444)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153106edead0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153106c89fd0>, 1555222.180061105)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153105f1c7d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302c9974d0>, 1555221.998191207)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153105e5d4d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056cccec0>, 1555222.670357907)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153106f3a8d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153106c890f0>, 1555222.481195321)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153106f390d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153102c4ba10>, 1555222.569771262)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153106f97e10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153106dc4280>, 1555222.222200015)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153106db57d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153105ee49f0>, 1555222.432866344)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153106ddfad0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153106dc54e0>, 1555222.849837304)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153106e446d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153106dc6dd0>, 1555223.880086305)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153106ea0d90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153106dc7000>, 1555223.622234661)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153106e53ed0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153106dc7f50>, 1555223.775105645)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153042c2e7d0>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153106c894e0>, 1555223.654266406)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153042c8d110>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153042c49be0>, 1555224.842359105)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153042cbb510>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153105ee65f0>, 1555222.654965205)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153106e41810>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153106165610>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530ffbddc50>, 1555214.296063838)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153106167c10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304350c950>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302d3daa50>, 1555227.564214727)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304299e390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304294fc10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153042907650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153106dc48a0>, 1555227.27434458)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fc54e790>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531076cf150>, 1555227.714240839)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153042906d10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153042ca06d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530431ccb50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530ffbdc360>, 1555228.257610549)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153042910050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302cb8df60>, 1555227.760687034)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530431cc990>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153043501850>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056ee4bb0>, 1555228.070559243)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153043500c90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153102d11750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303cf477e0>, 1555228.382903032)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153102d117d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531076f1090>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302d4c8ad0>, 1555228.742435117)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153043500f90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304294ee50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531060f5710>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302d4cad60>, 1555228.55287456)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304294d610>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056ee60b0>, 1555228.607078566)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531060f4f90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153105b96190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd6b7620>, 1555228.320240172)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153105b95510>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153106951450>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302cb8d2b0>, 1555228.516632523)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530feae3590>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530429dcfd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531072a1450>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153106dc74d0>, 1555228.890416593)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153043501c10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd8d5b70>, 1555228.714977362)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531072a1210>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15310771ba10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15310726b390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153042944a50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304294eb90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304295f2d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153042966ed0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304295e690>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304298a4d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530429bf190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530429b30d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531017544d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530429d1e90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530435f0650>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153106eef8d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530ffbdf850>, 1555227.334037615)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153105ead050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531064064a0>, 1555227.356760341)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153042944850>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd8d65f0>, 1555228.471354053)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15310771a690>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd8d5a90>, 1555229.07583751)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153107269010>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530429b91d0>, 1555226.592256354)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304294f610>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fcc7eeb0>, 1555226.887790137)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304295fb10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530429b9080>, 1555227.06381115)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530429675d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530429b9390>, 1555227.255710694)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153042967b50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531057d0590>, 1555226.743977254)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153042989c10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fc7ca040>, 1555226.9117525)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530429c8390>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531076ce7b0>, 1555226.767470606)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530429d0790>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302c9943d0>, 1555227.086835382)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153042967a50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302c997e00>, 1555227.160087685)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530429d1b10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056ccc1a0>, 1555227.133637569)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153042989890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531070df590>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd8d6660>, 1555229.340666888)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531070de450>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153105ec8090>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd8d49f0>, 1555229.137652101)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153106c09450>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15310208e290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531038d6ac0>, 1555229.947550964)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15310208c0d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531022e2250>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531071f4dd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531038d50f0>, 1555229.713607954)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531022e2490>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531038d4f30>, 1555229.849337064)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531020fe610>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531026d2890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531038d7e70>, 1555230.501492951)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531026d2c90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153102704150>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304297bbd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056ee6c80>, 1555229.764702497)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15310277d190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531038d6cf0>, 1555230.176040072)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15310237c9d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153102725890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531029bfe50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531038d4670>, 1555230.261321817)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153102727d90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153102b2f550>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15310389e4a0>, 1555230.774775654)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153102b2e710>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153102b1cb50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531044e96a0>, 1555230.07902528)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153102b1c250>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153105af4810>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153106133b90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153105954290>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531070fe210>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531044e8a60>, 1555230.199801672)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531029bfcd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531038d66d0>, 1555228.88874258)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531070fda50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302d4c99b0>, 1555230.936992514)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531059740d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd8d6eb0>, 1555228.942077474)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530431a0b90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd6b60b0>, 1555228.964311571)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153105954e90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fe9b5e50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530429060d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd8d4bb0>, 1555230.736291376)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153105919390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15310557df10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531044e9f60>, 1555231.286516477)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15310557d810>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153105504550>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153103d04210>, 1555231.315472697)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15310551da10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153102994950>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fccb2740>, 1555231.404864951)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153102a4ec10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531054cbd90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153103c00fa0>, 1555231.642759236)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531054cbf10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531054bb3d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153105c4ccd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153103c02f20>, 1555231.939488621)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531054b8e90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153103c00ad0>, 1555231.628468728)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153105c4d810>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153105c12550>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153103c00c90>, 1555231.951854961)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153105c10cd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153105c29050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153103c02ba0>, 1555232.507816896)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153105c2bad0>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15310625e950>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531038d5860>, 1555231.957662308)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15310625e650>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153042964a10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153106226810>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531038d4ec0>, 1555230.99025645)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531027e81d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153102abc510>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153102aefb90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153102aeb710>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153102a03e90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531044ea660>, 1555230.300980995)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153102abce90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531044e9010>, 1555230.603478407)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153102a01a50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531044ea4a0>, 1555230.653466292)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153102aeb5d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531038d5a20>, 1555230.58156026)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153042966f10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153103c02dd0>, 1555232.23738002)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531029ed350>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153103c03a80>, 1555232.327034669)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153106225b90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531061b4c10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303d0386e0>, 1555232.306337504)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531061b5890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153102760310>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15310637d150>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303d038d00>, 1555232.334866405)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15310267b450>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303d03a040>, 1555232.676753633)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15310637d310>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153102aeb690>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531062e8ed0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303d039a90>, 1555232.37121315)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153102aeb3d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd8d77e0>, 1555233.008640515)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531062e8350>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531065a6f90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153102a4d890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303d03b930>, 1555232.844267437)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153102a4c510>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153106517550>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15310654b850>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302cadcc20>, 1555234.214346219)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531065172d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302d4cbaf0>, 1555233.24670805)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15310654a350>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531064d3310>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302caddc50>, 1555233.547632668)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15310224b4d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531074a9010>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303d03bee0>, 1555233.303141453)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531074abd90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531063285d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302cadf620>, 1555233.427706939)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153105be28d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153107431cd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b9fe270>, 1555233.679198045)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15310535d850>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15310639e6d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531061f4290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153103d048a0>, 1555232.668962573)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153101750590>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303d038ec0>, 1555232.153565355)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15310639e310>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b9fd400>, 1555234.977444079)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153107430990>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531038d5da0>, 1555232.34649721)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531061f7410>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15310654b990>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153101d03c90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153103c007c0>, 1555234.038833388)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153101d03190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531052fe590>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531073b7d50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153105346110>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b9fcc90>, 1555234.269082695)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153105347950>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531053a9f90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153101cea0d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b9ffa10>, 1555233.944624412)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531053aa010>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b9fe190>, 1555234.164299816)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153101ce9b50>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153101c5f990>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153102750910>, 1555234.162274121)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153101c5d890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153101c70bd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057fdef20>, 1555236.349572599)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153101c700d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153106386650>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153101edd750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531044eaf20>, 1555235.479917378)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531054bb210>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b9fd080>, 1555236.849718205)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153101ede590>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153101e44710>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057fde200>, 1555236.390453904)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153101e47a90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153101eb8810>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd507ee0>, 1555234.898858193)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153101eb9250>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153101ef4410>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531038d5320>, 1555233.886393295)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15310630acd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153107430110>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b9fd8d0>, 1555234.464875684)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531053112d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153102c48750>, 1555233.891778285)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531073b7350>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303d03b9a0>, 1555235.291931301)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153101c4dbd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153103c01080>, 1555233.807741605)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153107430390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153101ef6ed0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd506b30>, 1555236.347750733)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153106522dd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153042967bd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd13ca60>, 1555236.706982078)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153105c11f10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153101fc3610>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153101f6f5d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b9fc210>, 1555237.284127015)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fd8a0990>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b9fc7c0>, 1555236.868668003)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153101f6f410>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153101fef250>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153101f37710>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303d4531c0>, 1555236.035800874)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153101d23c50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303d03ae40>, 1555236.376532857)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153101f37610>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531024e9950>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302ca395c0>, 1555237.279964334)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531024ea210>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153106516f10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153102023b10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302ca3a7b0>, 1555235.971162579)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15310294d490>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302ca391d0>, 1555237.002446411)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531062fe5d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d991a90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153103c034d0>, 1555236.091971017)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d9904d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153101ffc910>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302dafeb90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302ca38590>, 1555236.600935707)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153101ffd4d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd506120>, 1555236.210457501)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fd8e26d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530341270d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302ca38360>, 1555236.649602586)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153034125450>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530341e4690>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303d03b0e0>, 1555237.134858797)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530341e4150>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d9a8550>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153105bb8d50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd505d30>, 1555237.534375428)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d9a8d90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd5047c0>, 1555235.092312434)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153105bba2d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d722c90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d70e150>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302ca39ef0>, 1555237.860970234)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d721cd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fc9b6ba0>, 1555237.708907331)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d70cc90>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302dafc090>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303fcd5c50>, 1555237.621917642)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302dafc710>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d7f6a90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fc9b6dd0>, 1555238.400286463)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d7f7e90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531024e9ad0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d730dd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fc9b49f0>, 1555238.1004445)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531062fd350>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fc9b6580>, 1555238.877510976)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d732390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302db57990>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d8b98d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fc9b6f90>, 1555238.779007711)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302db55610>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fc9b5860>, 1555238.429896637)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d8b9410>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302dbc8b10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fc9b67b0>, 1555238.010215212)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302dbcaed0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302db3b310>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fc9b74d0>, 1555238.133823285)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302db3b190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d9d1650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b9fd1d0>, 1555238.996578478)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d84d750>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d742a10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d66ad50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fcc27700>, 1555239.13032206)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d641a50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fcc27540>, 1555238.795722281)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d66a090>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531024d0250>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fcc24a60>, 1555238.604578219)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531024d0950>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fd96e610>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fcc26190>, 1555238.670573979)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d6acb90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fcc26900>, 1555238.960192424)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d752210>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d6705d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd505400>, 1555239.293186007)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d6727d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d540110>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fcc27000>, 1555239.396906508)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d688510>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d7e6510>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fcc240c0>, 1555239.092306116)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d7e6b50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d64b8d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d474fd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fcc26ba0>, 1555240.226652466)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15310556f310>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fcc25630>, 1555239.304791955)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d4762d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d6dea50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fc3acd70>, 1555239.943058969)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d6dead0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302dbd0450>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057f08bb0>, 1555239.786233066)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d51d0d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d627450>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303fcd5b70>, 1555271.476971839)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d481250>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d4fa1d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d521150>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fcc24050>, 1555240.953815584)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d4f9450>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057f08670>, 1555239.506904816)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d523290>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d5eba50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fc0a2c10>, 1555240.452327836)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d5ea350>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d462ed0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057f08590>, 1555240.725237488)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d426710>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d379cd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fcc277e0>, 1555240.214084216)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d3797d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d8a4d90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531044e9860>, 1555240.628331526)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d4d4c50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fc9b4de0>, 1555240.659550812)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d467c50>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d9c6e50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d5aa150>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531038d5e10>, 1555273.087927692)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d592b90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fcc7d9b0>, 1555272.879419404)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d5a9810>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d412050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057f09940>, 1555272.113982516)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d54b3d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302caf7010>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302cc9aad0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fcc7d390>, 1555271.29617679)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302caf7fd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fcc7cec0>, 1555273.10317516)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302cc98690>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302cc03010>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302cccbd50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fcc7ce50>, 1555272.743724026)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302cca68d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d3a4a50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056b04a60>, 1555271.132288856)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d3a52d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302cc44e10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056b079a0>, 1555273.008815148)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302cc46850>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d464690>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302cb02050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056b066d0>, 1555272.893026692)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d7a7190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302cb64350>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d4e55d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d489d50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302cc166d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057fdc3d0>, 1555272.143452137)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302ccc8a50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056b04600>, 1555273.063405217)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302cb01550>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fcc7f850>, 1555240.624432154)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302cc16e50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057f083d0>, 1555271.139857552)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302cb64c90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fcc7d2b0>, 1555240.810642624)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d4e7250>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fcc7e510>, 1555240.935303992)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d48bcd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153054c79910>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056b07e00>, 1555274.244376729)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302cbb8650>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302ca8ced0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fcc264a0>, 1555273.102016068)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302ca8e510>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530499eafd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302d5b59b0>, 1555272.082088624)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530499ea810>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153034d9d510>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd758210>, 1555273.222191864)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303d955890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303dd8f6d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd75bd90>, 1555273.354385952)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303dd8f690>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302ca1fd90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302cbd52d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302ca80f50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302cb5e510>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302d5b5400>, 1555274.566891515)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d463f90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302da0cd10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303dc50050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fc9b5010>, 1555274.240675134)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302ca1ddd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056b075b0>, 1555274.470387878)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302cbd7210>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303f1edbe0>, 1555273.341946819)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302ca83b50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd75ae40>, 1555274.37076551)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302da0d8d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd7596a0>, 1555274.901867045)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303dc51cd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303d4aa690>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302c83eba0>, 1555274.598524707)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303d4a8c10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153034157310>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056b06b30>, 1555274.573640732)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303d1c2e90>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153035050e10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302c7f4e50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056c50fa0>, 1555275.860479725)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153035051250>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b8543d0>, 1555275.724287449)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530573b5c50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530497a63d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b854210>, 1555276.983096819)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303fc2f3d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303518d190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b8572a0>, 1555276.379564755)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d31f1d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303db74950>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302ccdea50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302caeef50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302c873150>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303d7d9f90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303d7cd5d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056c50f30>, 1555275.826176756)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302cc9cfd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057f0b380>, 1555274.897938409)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302ccdc850>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056b05e80>, 1555275.983418885)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302caed9d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b856120>, 1555274.45093389)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302c870250>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fcc7edd0>, 1555276.125477004)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303d7da510>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b854670>, 1555276.524991087)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303d7cffd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153034185dd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303dfae910>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056c51da0>, 1555275.923201847)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153034186f50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b855550>, 1555275.65943665)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303dfaf550>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153034fda490>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153035853d10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153105246c80>, 1555277.764273649)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153035852ad0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302db64410>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153034df1790>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153105245470>, 1555276.578233359)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302db67610>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153105244600>, 1555276.48776399)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153034df0110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153034cf66d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302d34faf0>, 1555276.799491336)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153034cf6690>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530350fba90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302d34c4b0>, 1555277.930715334)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530350f9e10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530555784d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302d5267b0>, 1555278.247007688)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303e43eed0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302ca3da10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302d34cf30>, 1555277.46894957)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153034114d10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fdbf8c10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530493614d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302d44b0e0>, 1555277.824285187)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fdbfbad0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530491b9f50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057f0a970>, 1555276.306406118)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153034fd8050>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153035379f90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153034c85890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153034cc7e10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b857540>, 1555278.085962814)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153049361450>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302d526eb0>, 1555276.000568442)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153034cc6190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302d449a90>, 1555277.774187113)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530491b9ed0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fcee39a0>, 1555275.711242408)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153035379490>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531052464a0>, 1555276.846957727)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153034c87910>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153034f87e50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302d34f5b0>, 1555278.022073676)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153034f87e10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303e516f90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303fc0cfd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056b06f90>, 1555277.535594025)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303e514410>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302d4498d0>, 1555278.292227723)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302cbe7190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153055c40e50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fcc27d90>, 1555278.282377541)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153055c41d90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303dac6910>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fcc7f000>, 1555279.520917506)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15305418c150>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530561a38d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153056e2d4d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303d25e550>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b65b1c0>, 1555278.825691838)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303d1ce0d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303d295c10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b659cc0>, 1555279.227630402)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303d2966d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303cf15890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302d34e120>, 1555279.260051244)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302c957c50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303d73a310>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b659400>, 1555279.852010322)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302ca3d890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303d7c9590>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b658f30>, 1555279.246377869)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303d7c9990>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530557b77d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302d44b770>, 1555278.017665278)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153056133650>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303cf29c90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303cf1ea50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153105246f90>, 1555279.231706441)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303d25e850>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b856430>, 1555278.994344114)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153049734a10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303f1ed550>, 1555278.900460204)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303cf28c10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b658830>, 1555278.932515174)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302cbd4890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302ca831d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530557b42d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b659f60>, 1555282.360874715)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153048d35590>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b857c40>, 1555279.877409701)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302cb2b890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303d514550>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fc7cbc40>, 1555280.228452104)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303cf1cf50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303d877a90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153105245940>, 1555279.782881668)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303d874bd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530ff5c8810>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303d8f7a90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fc7caa50>, 1555279.760507568)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302c7fd950>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fc7c8210>, 1555280.537606199)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303d970590>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303dae41d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fc7c9400>, 1555279.88475799)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303dae6550>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303dab1c10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303dd75610>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302d44add0>, 1555280.605621139)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303dab0e90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302cc70c20>, 1555281.290674982)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303d7fa8d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303da573d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302cc72430>, 1555280.864079006)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303cef00d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303de9bc10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153105247f50>, 1555280.378414079)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303d14add0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153105223090>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fd1193d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303dbf77d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302cc73540>, 1555280.653263643)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fd281510>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b65bbd0>, 1555280.692781583)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303dbf5d50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fc928d90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302cc718d0>, 1555281.856941794)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303e05f410>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fc7dba10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303df88bd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b65ba10>, 1555281.113575558)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153105222950>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302cc72e40>, 1555281.183771257)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fc7db110>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302cafdc50>, 1555281.923135174)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303df88390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303df0cdd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302caffc40>, 1555281.531763917)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303e114310>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303da83590>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fc97f8d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302d449010>, 1555281.460955679)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303da83b90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302cafda90>, 1555281.720076791)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fc97f410>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fd2d22d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302d449400>, 1555281.659655538)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fd2d3590>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153057aefdd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303e23e550>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302cafcc20>, 1555281.86420689)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153057aef850>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fc7cac80>, 1555281.890127597)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303e23dfd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153107766350>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fcf88110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153107747dd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd222e40>, 1555283.016792008)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153107747e10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fc411d10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd223540>, 1555282.259732889)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530579c0cd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303de991d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fc622d50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd221080>, 1555282.536441961)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fc8baf10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd2229e0>, 1555283.283410369)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fc623450>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15305666b990>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fd377b10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fc131c50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fcc7e3c0>, 1555282.057755787)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153107767e50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302cc708a0>, 1555282.002587678)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fcf8b890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b856cf0>, 1555283.84345661)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fc630b10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd220a60>, 1555282.798753711)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fd374ed0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302cafe660>, 1555280.96514264)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303e3b7190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153034fd9250>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304be48910>, 1555283.181957788)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303e67ded0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153107398210>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15310761d7d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd223380>, 1555283.276399321)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15310739bd90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153105246430>, 1555283.201570826)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15310761f390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303da80dd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fe52e210>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd2227b0>, 1555284.947129593)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fcaf7310>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304be4bd20>, 1555283.837138786)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fe52eb90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303e05c2d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153106a0a310>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302cafd0f0>, 1555284.640665412)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303d80f6d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304be49470>, 1555283.584082743)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153106a09790>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302db94390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153106aa66d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b65b460>, 1555283.492546221)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302db96110>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304be4a430>, 1555283.783490925)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153056cef390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302dad6010>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303fce7b60>, 1555283.904432981)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302dad5310>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302dba1310>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303fce6190>, 1555283.863485697)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302dba3c10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d5a2b10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303df53b50>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302cc72ba0>, 1555284.123143524)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15310648d410>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303fce7d90>, 1555284.156435834)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303df51b50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303cf3db90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fd566d50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d387350>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd2216a0>, 1555284.334325797)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303cf3e4d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd222120>, 1555282.697526052)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fd566dd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304be48c90>, 1555282.709057973)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d387e50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153106d00250>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153043513e70>, 1555284.468458959)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531060ddf10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153105865110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153106da0b10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd2226d0>, 1555284.507674101)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153105865290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530435119b0>, 1555286.138660651)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153106da3110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303dcf7190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303dfc1810>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b854c20>, 1555286.18598993)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fd567c10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd2208a0>, 1555285.73724893)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303dfc2f90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153107093610>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153104f178c0>, 1555285.917778383)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153107093850>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303e5a2bd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153104f16d60>, 1555286.060404461)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304284f250>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303e265e10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153104f17850>, 1555286.094882216)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153106b09d90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fc662150>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302cafd2b0>, 1555286.064779814)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531059c02d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153106b683d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153106b64790>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304be48b40>, 1555286.093733761)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302db83cd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153104f175b0>, 1555285.441281696)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153106b64550>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153106b17c50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd223000>, 1555286.213515184)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153106b15b90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fc633690>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15310737e390>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b65b3f0>, 1555285.535315804)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153106646b50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153104f15cc0>, 1555285.551070842)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15310737f790>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15310704b090>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530431943d0>, 1555285.806103471)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531070497d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531071dad50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153043195fd0>, 1555287.520593168)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153106d9f990>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153106d2ac90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153043196430>, 1555287.77429694)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153106d2bf50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153106d9a990>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd223230>, 1555287.216136676)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153106d990d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153107392190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530435109f0>, 1555287.926362617)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153107390a10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531061cd690>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd220910>, 1555287.093582285)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531070ec450>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153106afe8d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531043d2660>, 1555287.5433961)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153106afd050>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303dc10750>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153106cf6050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531043d2ba0>, 1555287.490503319)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303dc11d90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531043d1240>, 1555287.436802685)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153106cf7e90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153106ff4a90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153043511160>, 1555287.683262841)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15310648d310>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153106af2250>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153106d35090>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153106ce1d90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153043197c40>, 1555286.982845521)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153106ce22d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302caff1c0>, 1555287.974423896)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531074e52d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302d44bee0>, 1555286.013345931)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153106d37d90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d43f650>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153106d71d90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd220de0>, 1555288.162522231)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531070154d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531074fafd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531074fb750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153043512120>, 1555288.423636034)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531074f83d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530488b87c0>, 1555288.127681526)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531074f9e90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153107696810>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530488b86e0>, 1555288.921349668)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531072e2dd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303dc73890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531075dfa90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304be48fa0>, 1555289.181185833)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531061cd890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530488b81a0>, 1555289.144236151)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531075dcb10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531075f54d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531043d12b0>, 1555288.434986784)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531075f57d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531057fd4d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153107562890>, 1555288.956869319)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531057ffe90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153107534d10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153102218990>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531075631c0>, 1555289.664130331)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15310221dc10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153107560d70>, 1555289.829556797)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153102218650>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15310586d1d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153106b17f10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531073558d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153104f14280>, 1555288.280313148)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d43d650>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531074bf010>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531076ab490>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531043d0600>, 1555287.93052347)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153107354090>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530488ba0b0>, 1555289.295646197)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531074be690>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530488ba890>, 1555288.683285397)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531076a89d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153043196190>, 1555287.787155189)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153106b15b50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153107560830>, 1555289.597916005)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15310586c550>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fcf8a310>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530488b8130>, 1555288.944497367)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153107372450>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153107534fd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153104f144b0>, 1555289.740302554)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153107537310>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15310583d9d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15310214bad0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531075635b0>, 1555289.946141147)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531028abc10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153107561550>, 1555290.017761708)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15310214b910>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153107578e50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15310289c890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531075618d0>, 1555289.464777609)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531057ed3d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531043d1630>, 1555290.187765682)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15310289f1d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531028bd850>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153043197460>, 1555290.352125257)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531028bd550>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153102838e50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048974600>, 1555289.956663538)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15310283a010>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531028876d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153105d76ed0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530489772a0>, 1555290.509437125)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531074fa390>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048977230>, 1555289.947405233)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153105d75610>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153105ccaf10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153107560750>, 1555290.503193862)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153105ccb210>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531021bcc10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153107561f60>, 1555291.248992527)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531021bf5d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153105d416d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153105ce6510>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048977c40>, 1555290.57420237)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153105d401d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153107560f30>, 1555290.990389598)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153105ce78d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531064495d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048976040>, 1555290.740859683)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531064499d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153106489e90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531074bc910>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048974c90>, 1555291.186076674)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15310649a910>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531075626d0>, 1555289.72165099)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531074bf5d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15310642c790>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530488b9a90>, 1555291.06287439)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15310642d990>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531063f9a10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530431973f0>, 1555291.388461794)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153107627610>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531077de250>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048b25630>, 1555291.284795974)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531021bc290>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15310787d890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048b27150>, 1555291.460362605)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15310787ec10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153106427d90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15310783cfd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048b24440>, 1555291.935011659)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153106426f10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048b26510>, 1555291.667378029)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15310783d610>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15310219e8d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048b259b0>, 1555291.657582314)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15310219e610>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15310580fdd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153101bc6490>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048b27d20>, 1555291.539112974)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15310580cdd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048b27070>, 1555292.141397833)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153107666190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153101bbce90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153101b61b10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048b25470>, 1555291.783599122)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153101bbd910>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b5592b0>, 1555292.29408491)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153101b63750>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153104099090>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b559710>, 1555291.809681549)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153104098ad0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531040531d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b55bbd0>, 1555291.989923946)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153104052ed0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531064493d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531040d9a90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b558f30>, 1555292.193862351)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531064484d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b5587c0>, 1555292.870337074)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531040d9c90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302c7d8050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b55af20>, 1555292.420929996)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302c7d8390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302c797190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b55b8c0>, 1555292.739294292)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302c7a9390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304b1a1590>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b558e50>, 1555293.175887257)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304b1a0110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304b1fb810>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153107616b30>, 1555292.688734216)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304b1f9e50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304b1f2150>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153100b8ef90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153043196d60>, 1555293.254017366)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304b1f2a10>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530488ba3c0>, 1555292.980728375)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153100b8fb90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153100bb4b50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153107615080>, 1555293.247999147)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153105ce4dd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153100b7a090>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531076144b0>, 1555293.115577717)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153100b78dd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304b1a89d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302c8bb9d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153107616900>, 1555293.067867231)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304b1aaa10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153107616740>, 1555293.551338936)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302c8b8f10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302c735c90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153107616f20>, 1555293.255774308)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302c734490>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153100bbd950>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531076171c0>, 1555293.881806836)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153100bbc2d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303d8b9050>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302da52710>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153107617770>, 1555293.71359873)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303d8bbb50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531076154e0>, 1555293.244921575)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302da516d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531057da2d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048b26970>, 1555293.487410418)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302c73bdd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531057d81d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048b246e0>, 1555293.275615269)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531057dbb50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153101a4a550>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303d142200>, 1555293.875847692)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153101a4a390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304b191c50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153101b5f090>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303d140590>, 1555294.054401353)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531069e9cd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303d141b00>, 1555294.710260463)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fe44ec90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302c789b90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303d141710>, 1555294.258396762)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302c7886d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304b1f3310>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303d143310>, 1555294.479747889)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304b1f2890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153101dc54d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b55a970>, 1555294.81288124)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153107572090>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302da34350>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bcb1470>, 1555295.067865012)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302da50310>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153101eedb90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153102200e10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530489760b0>, 1555294.525094482)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153101eef850>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bcb31c0>, 1555294.804868191)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153102203bd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153101e94250>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bcb3700>, 1555294.77310117)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153101ed9f10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153100b8fe90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153102251190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bcb1080>, 1555296.430688183)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153100b8f410>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153102085390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531063728d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153102382950>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bcb1160>, 1555294.762391201)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153102084690>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bcb0520>, 1555295.123848215)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153106373950>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bcb02f0>, 1555296.377057962)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153102383d10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153101b7a5d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bcb2890>, 1555296.298779069)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153100b70690>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303d140a60>, 1555293.708797082)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302c771d10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153101b61350>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15310267fb90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303d357070>, 1555295.617121924)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153101b61a90>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153101fb5350>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302c6ef4d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303d354bb0>, 1555295.68334849)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302da36650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303d354980>, 1555295.407770647)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531027166d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15310262b610>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153102746590>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303d1416a0>, 1555295.608095238)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153102628890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303d3572a0>, 1555295.626841455)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153102744710>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153105eecb10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303d142dd0>, 1555296.758236209)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153101bc7590>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531026ecd10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303d3549f0>, 1555296.312144444)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531026ec390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153102b4ced0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bcb3a10>, 1555296.887131006)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153102b4e3d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153102116d50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303d3579a0>, 1555296.437925425)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153102116850>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153102c23790>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153105c2ced0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302c6f9860>, 1555296.918883907)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15310289c810>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302c6fa9e0>, 1555296.661123703)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153105c2ee50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153105392bd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bcb09f0>, 1555297.147882191)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153105393850>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531052bd790>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531066f4450>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302c6f97f0>, 1555296.436764855)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153106372fd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302c6fb380>, 1555297.825423292)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531066f7010>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302c6f98d0>, 1555298.113797733)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153105585690>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bcb07c0>, 1555296.681102117)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15310267f090>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15310615ded0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd18db00>, 1555298.429546978)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153105dac750>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153105ae5510>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd18e890>, 1555298.669776053)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153105ae7cd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15310598cc10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153105b29590>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd18fd20>, 1555298.342857809)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15310598db50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd18e120>, 1555297.904613734)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153105b28550>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153106b53d10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153105ee6820>, 1555298.275218096)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153105a5e6d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531052fad10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fce75a90>, 1555298.36906851)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531052f9d90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153106711050>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15310655d390>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fce75c50>, 1555298.21173583)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153106711650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fce74670>, 1555298.467163535)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15310655d350>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530429c84d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15310620b650>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153102577010>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153105c6eed0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15310611d4d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303d3556a0>, 1555297.690793234)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153102591450>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302c6f9d30>, 1555296.762185544)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153106209f90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303d357620>, 1555298.949457023)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530429c8150>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd18df60>, 1555300.082640532)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153105c6ffd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd18eba0>, 1555299.732165564)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15310611e850>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153105fa7490>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303d355c50>, 1555299.004569754)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153105fa4810>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530430aea90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fce74a60>, 1555299.060423987)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530430ae950>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530430d6fd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153106c8a9e0>, 1555299.287463274)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530430d6e90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530430b5350>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153106c895c0>, 1555299.660817128)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530430ad190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153042e02e90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fca3e660>, 1555299.628987787)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153042e02d50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153042e364d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153042e53890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531067deed0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bcb20b0>, 1555300.26840047)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531067dc410>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153042e75e10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bf1e190>, 1555332.367841385)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530430ede50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153042e8bad0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153042e8f250>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530429f8a10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530429a6110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153100b8e610>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530430966d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fce74f30>, 1555299.430224703)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531061f0250>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bcb1b00>, 1555299.978753361)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153042e362d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fca3e200>, 1555300.598264313)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153042e53690>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303d3552b0>, 1555298.597925326)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153043097e50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bf1f070>, 1555301.076115863)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153042e8b990>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bf1e270>, 1555301.131378336)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153042e8f110>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fce75940>, 1555298.354367086)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530429a7850>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302c6f99b0>, 1555299.715332393)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153102b15890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153042ea2210>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153042eaa9d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fce77a80>, 1555332.321990045)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153042ea2110>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bf1cf30>, 1555300.723294226)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153042eaaed0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153042efaad0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bcb0130>, 1555300.767960973)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153042efb190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153042d0a110>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bf1fd90>, 1555300.96234711)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153042d09fd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530430d6f10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bf1db70>, 1555300.844770774)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530430d6950>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153042d0b810>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056e79c50>, 1555331.38204846)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153102ab32d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153042d58f90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd1d9da0>, 1555331.195508812)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153042d41550>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530430e12d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153042d91fd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153105036820>, 1555332.357988857)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153042d404d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153105034210>, 1555331.383664019)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153042d90090>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153042d43790>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153042db9310>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bcb3cb0>, 1555333.560760364)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153042d688d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153105035da0>, 1555332.13562404)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153042db90d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153042e32390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153042de9fd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153106c8bd20>, 1555332.861950912)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531067dd650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153105035a90>, 1555331.29744876)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153042de9dd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153042d5ab10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153042eda810>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153042eccc10>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153105036eb0>, 1555332.183395476)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153042edbc10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bf1d5c0>, 1555300.419023222)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153042ed8950>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd18e270>, 1555300.637680908)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153042eccbd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153042ebca10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bcb0750>, 1555332.197710566)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153042d392d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153042bd1310>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531057d3e70>, 1555332.209279421)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153042d5a410>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153042bf42d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bf1dd30>, 1555335.171570299)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153042bc4890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153042a9de90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153042dfb1d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530430d7e50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153042b3d350>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153042b60990>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153042b7f890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153042badd10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531057d2580>, 1555333.773832785)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153042bad990>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153105036890>, 1555331.420106198)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153042b3d890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bf1ed60>, 1555333.889346505)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153042a9dc90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bf1cd70>, 1555333.465843499)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153042dfa4d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fce76cf0>, 1555332.205818424)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530430d6c90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531057d1a20>, 1555331.172799898)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153042b607d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531057d3380>, 1555333.472932307)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153042b7cf90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153042aa6b50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531057d1630>, 1555333.86181458)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153042aa6a90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153042adc2d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153105ce37e0>, 1555335.301283771)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153042adb790>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153100e80790>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153105ce19b0>, 1555335.31302758)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153100e81090>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153100e9f610>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153105ce1ef0>, 1555335.113499871)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153100e9de10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153042bc49d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153042aba950>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153100e38650>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153100e5d050>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153100ea2950>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153100ed0310>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153100eeed10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153100f071d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fca3e900>, 1555335.307434038)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153042bc7850>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153105036120>, 1555335.523686304)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153042abafd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153105ce0d00>, 1555335.238116875)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153100e38d10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531028f8130>, 1555335.176887172)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153100e5d910>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153105ce12b0>, 1555334.943255425)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153100ea3150>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153106406eb0>, 1555335.242830265)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153100ed0ad0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153106407d90>, 1555335.080219816)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153100eeeb10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531057d0520>, 1555335.30019212)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153100f05890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153042ad4310>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fce74210>, 1555336.061761505)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153042d249d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15310165bf50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153106405780>, 1555336.776670201)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15310165be50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15310166f190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15310780baf0>, 1555336.685723747)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15310166f050>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153042baefd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531078098d0>, 1555336.892735009)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153100e9e410>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153100f43750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531028fa970>, 1555337.286946687)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153100f43d50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153042a9df10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531028fa9e0>, 1555336.866072983)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153042a9f690>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153100f57f10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153100f85890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303d357bd0>, 1555336.843723453)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153100f57c50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153104019b00>, 1555336.58412796)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153100f85690>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153101706290>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153100fa05d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153101639090>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15310163df90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153101698dd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153100eba990>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15310780a120>, 1555337.011025667)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153101698c50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531016d0790>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15310780b850>, 1555336.424879648)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153100eba850>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531028fa3c0>, 1555335.389911634)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531016397d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531028f86e0>, 1555335.404588423)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15310163dd90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15310780b690>, 1555337.421561322)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153101707810>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15310780aa50>, 1555337.301886695)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153100fa03d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153107809710>, 1555337.131650114)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153042b60cd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153100f56850>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15310401bee0>, 1555337.617078029)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153100f565d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153100ffcd10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15310401a040>, 1555338.07713493)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153100ffcb10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153101025990>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531028fb770>, 1555337.529038352)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15310100e4d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153042f24150>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531040182f0>, 1555337.913414378)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153042bf4890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153042f4a7d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153104018f30>, 1555338.399124284)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15310165ad10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153042f79990>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302c78e350>, 1555338.264855769)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153042f797d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153042e03e90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153105ce2dd0>, 1555338.963332604)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153101699690>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153042ad40d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153042fb8610>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153107809320>, 1555338.749407118)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153042ad62d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302c78f460>, 1555338.6429356)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153042fb7b90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153042fbb710>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153042af4990>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153042aa7590>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153100fc84d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15310100fbd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153042a40310>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531064065f0>, 1555338.27373456)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153100fc8c90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531064062e0>, 1555338.310219641)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530430e0a10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153106405860>, 1555338.14296103)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153042af5590>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153104019d30>, 1555337.363336569)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15310100fb90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302c78d550>, 1555338.840146403)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153042fbb4d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531064046e0>, 1555337.978285535)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153042e1aa10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153101004f90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153106407e70>, 1555338.894896752)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153101004a10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153042fabcd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15310780bd20>, 1555340.109442526)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153100f57550>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153042fed4d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531057d0360>, 1555340.029537242)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153042fed410>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153042a96f90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531018d5310>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531057d2900>, 1555339.034660753)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153100fa1950>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302c78c4b0>, 1555339.023579724)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531018d5890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153100fe0650>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153101900690>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531050374d0>, 1555339.162902564)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153100fe0e90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15310780a9e0>, 1555339.368635784)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153101900d90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153042f25e10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15310780ba10>, 1555339.64967806)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153042f27f50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531018e7f50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153100b774d0>, 1555339.743289978)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15310192d890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153101862110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531016b0550>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15310401a190>, 1555341.872005591)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153101860650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153100b74280>, 1555339.810439478)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15310186ff90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153101891090>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153100b749f0>, 1555340.529729947)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153101890e90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153101893110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531018a7110>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153107808de0>, 1555339.935778671)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153101892fd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153100b77310>, 1555339.964278084)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531018a6fd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531018c2f90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153042ff9ed0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153100b74600>, 1555340.404516431)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531018c2d90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531057d07c0>, 1555338.794619159)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153042ffa610>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153100d99850>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15310401bc40>, 1555341.584541221)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153100d99690>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153100da0d90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530575f4e50>, 1555340.398824929)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153100da3a50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531018e6ad0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153100dd5d50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153100b75630>, 1555340.674985194)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153100db5890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530575f5860>, 1555341.477890338)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153100f3e390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153100dd9690>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153100defe90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153100b77a10>, 1555342.771887938)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153100d81c50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530575f79a0>, 1555341.289349009)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153100defd50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153100e19910>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530575f5320>, 1555340.626277429)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153100e19c50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153100e0cd10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530575f6dd0>, 1555341.063999307)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153100e0d990>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153101034f10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530575f5b70>, 1555341.695898869)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153101035050>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15310105f490>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530570944b0>, 1555341.549473784)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15310105f350>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15310186df90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153101081e50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057094520>, 1555342.107110532)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15310186fb10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057096a50>, 1555341.452219054)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153101081d10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531010500d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531010a8ad0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153100d815d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153100d5e2d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153100b76200>, 1555342.188876453)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153100d5c290>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153107809ef0>, 1555341.577409265)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153100d81450>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057096120>, 1555341.322556655)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153101050510>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057094600>, 1555341.650411237)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531010a9190>\n",
      "Evaluating workflow:  82%|████████▏ | 2448/3000 [09:02<27:51,  3.03s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  82%|████████▏ | 2450/3000 [09:02<20:06,  2.19s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "\u001b[32m2026-01-04 10:53:54.277\u001b[0m | \u001b[31m\u001b[1mERROR   \u001b[0m | \u001b[36mevoagentx.workflow.workflow\u001b[0m:\u001b[36masync_execute\u001b[0m:\u001b[36m104\u001b[0m - \u001b[31m\u001b[1mAn Error occurs when executing the workflow: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 1 second. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  82%|████████▏ | 2457/3000 [09:03<08:38,  1.05it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  82%|████████▏ | 2459/3000 [09:04<08:13,  1.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  82%|████████▏ | 2461/3000 [09:04<06:32,  1.37it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  82%|████████▏ | 2463/3000 [09:04<05:08,  1.74it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  82%|████████▏ | 2468/3000 [09:06<03:32,  2.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  82%|████████▏ | 2470/3000 [09:06<02:44,  3.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  82%|████████▏ | 2472/3000 [09:06<02:19,  3.78it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  82%|████████▏ | 2474/3000 [09:06<01:56,  4.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  83%|████████▎ | 2477/3000 [09:07<01:44,  4.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  83%|████████▎ | 2482/3000 [09:07<01:14,  6.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  83%|████████▎ | 2486/3000 [09:07<00:53,  9.58it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  83%|████████▎ | 2490/3000 [09:08<00:44, 11.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  83%|████████▎ | 2493/3000 [09:08<00:39, 12.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  83%|████████▎ | 2497/3000 [09:09<01:14,  6.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  83%|████████▎ | 2500/3000 [09:09<00:56,  8.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  84%|████████▎ | 2505/3000 [09:09<00:42, 11.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  84%|████████▎ | 2508/3000 [09:10<00:40, 12.28it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  84%|████████▎ | 2510/3000 [09:10<01:13,  6.63it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  84%|████████▍ | 2519/3000 [09:11<00:35, 13.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  84%|████████▍ | 2522/3000 [09:11<00:36, 13.06it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  84%|████████▍ | 2525/3000 [09:11<00:31, 15.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  84%|████████▍ | 2528/3000 [09:12<01:01,  7.72it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  84%|████████▍ | 2530/3000 [09:12<01:05,  7.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  84%|████████▍ | 2532/3000 [09:12<00:58,  7.96it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  85%|████████▍ | 2536/3000 [09:13<00:45, 10.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  85%|████████▍ | 2540/3000 [09:13<00:55,  8.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  85%|████████▍ | 2542/3000 [09:14<01:00,  7.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  85%|████████▍ | 2548/3000 [09:14<00:39, 11.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  85%|████████▌ | 2550/3000 [09:14<00:36, 12.48it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  85%|████████▌ | 2552/3000 [09:14<00:42, 10.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  85%|████████▌ | 2554/3000 [09:15<00:49,  9.04it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  85%|████████▌ | 2558/3000 [09:15<00:41, 10.63it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  85%|████████▌ | 2560/3000 [09:15<00:37, 11.74it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  85%|████████▌ | 2562/3000 [09:16<00:50,  8.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  85%|████████▌ | 2564/3000 [09:16<00:44,  9.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  86%|████████▌ | 2569/3000 [09:16<00:53,  8.06it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  86%|████████▌ | 2573/3000 [09:17<00:43,  9.82it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  86%|████████▌ | 2576/3000 [09:17<00:33, 12.71it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  86%|████████▌ | 2578/3000 [09:17<00:35, 11.96it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  86%|████████▌ | 2582/3000 [09:18<01:10,  5.97it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  86%|████████▌ | 2587/3000 [09:19<00:43,  9.58it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  86%|████████▋ | 2591/3000 [09:19<00:40, 10.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  86%|████████▋ | 2594/3000 [09:19<00:45,  8.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  87%|████████▋ | 2598/3000 [09:20<00:48,  8.35it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  87%|████████▋ | 2600/3000 [09:20<00:46,  8.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  87%|████████▋ | 2605/3000 [09:20<00:33, 11.77it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  87%|████████▋ | 2609/3000 [09:21<00:27, 14.05it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  87%|████████▋ | 2613/3000 [09:21<00:41,  9.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  87%|████████▋ | 2617/3000 [09:22<00:34, 11.06it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  87%|████████▋ | 2619/3000 [09:22<00:32, 11.66it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  87%|████████▋ | 2621/3000 [09:22<00:46,  8.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  88%|████████▊ | 2626/3000 [09:23<00:33, 11.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  88%|████████▊ | 2630/3000 [09:23<00:24, 15.29it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  88%|████████▊ | 2632/3000 [09:23<00:34, 10.52it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  88%|████████▊ | 2637/3000 [09:24<00:32, 11.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  88%|████████▊ | 2639/3000 [09:24<00:35, 10.07it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  88%|████████▊ | 2641/3000 [09:24<00:36,  9.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  88%|████████▊ | 2643/3000 [09:24<00:39,  9.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  88%|████████▊ | 2648/3000 [09:25<00:32, 10.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  88%|████████▊ | 2650/3000 [09:25<00:31, 10.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  88%|████████▊ | 2652/3000 [09:25<00:33, 10.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  89%|████████▊ | 2656/3000 [09:25<00:31, 10.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  89%|████████▊ | 2658/3000 [09:26<00:27, 12.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  89%|████████▉ | 2663/3000 [09:26<00:25, 13.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  89%|████████▉ | 2667/3000 [09:27<00:34,  9.58it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  89%|████████▉ | 2672/3000 [09:27<00:31, 10.47it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  89%|████████▉ | 2680/3000 [09:28<00:22, 14.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  89%|████████▉ | 2683/3000 [09:28<00:27, 11.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  90%|████████▉ | 2685/3000 [09:28<00:28, 11.11it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  90%|████████▉ | 2687/3000 [09:29<00:43,  7.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  90%|████████▉ | 2689/3000 [09:29<00:41,  7.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  90%|████████▉ | 2693/3000 [09:29<00:30, 10.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  90%|████████▉ | 2698/3000 [09:29<00:21, 13.77it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  90%|█████████ | 2700/3000 [09:30<00:54,  5.48it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  90%|█████████ | 2705/3000 [09:31<00:34,  8.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  90%|█████████ | 2711/3000 [09:31<00:21, 13.32it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  91%|█████████ | 2716/3000 [09:32<00:37,  7.66it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  91%|█████████ | 2721/3000 [10:02<12:00,  2.58s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  91%|█████████ | 2723/3000 [10:02<08:57,  1.94s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  91%|█████████ | 2726/3000 [10:02<05:48,  1.27s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  91%|█████████ | 2728/3000 [10:04<05:35,  1.23s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "\u001b[32m2026-01-04 10:54:56.175\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n",
      "Example: {'question': \"You are an expert in gene expression analysis. For K562 cells, assess whether perturbation of CSNK1A1 is associated with a significant change in PIGX expression compared with unperturbed controls. Answer strictly as 'Final Answer: Yes' or 'Final Answer: No'.\", 'answer': 'Yes', 'question_new': \"You are an expert in gene expression analysis. For K562 cells, assess whether perturbation of CSNK1A1 is associated with a significant change in PIGX expression compared with unperturbed controls. Answer strictly as 'Final Answer: Yes' or 'Final Answer: No'.\", '_id': 9915}\n",
      "Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 1 second. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  91%|█████████ | 2732/3000 [10:05<03:15,  1.37it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  91%|█████████ | 2735/3000 [10:05<02:17,  1.93it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  91%|█████████▏| 2739/3000 [10:06<01:45,  2.47it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  92%|█████████▏| 2745/3000 [10:06<00:51,  4.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  92%|█████████▏| 2747/3000 [10:07<01:03,  3.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  92%|█████████▏| 2751/3000 [10:07<00:43,  5.66it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  92%|█████████▏| 2756/3000 [10:08<00:27,  8.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  92%|█████████▏| 2762/3000 [10:08<00:15, 15.38it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  92%|█████████▏| 2765/3000 [10:09<00:38,  6.17it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  92%|█████████▏| 2772/3000 [10:09<00:22, 10.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  92%|█████████▎| 2775/3000 [10:09<00:19, 11.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  93%|█████████▎| 2778/3000 [10:10<00:17, 12.37it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  93%|█████████▎| 2780/3000 [10:10<00:19, 11.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  93%|█████████▎| 2782/3000 [10:10<00:28,  7.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  93%|█████████▎| 2786/3000 [10:11<00:24,  8.58it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  93%|█████████▎| 2788/3000 [10:11<00:22,  9.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  93%|█████████▎| 2794/3000 [10:11<00:17, 11.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  93%|█████████▎| 2796/3000 [10:12<00:30,  6.60it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  93%|█████████▎| 2799/3000 [10:13<00:29,  6.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  94%|█████████▎| 2808/3000 [10:13<00:11, 16.14it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  94%|█████████▎| 2811/3000 [10:14<00:25,  7.27it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  94%|█████████▍| 2813/3000 [10:14<00:27,  6.84it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  94%|█████████▍| 2816/3000 [10:14<00:21,  8.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  94%|█████████▍| 2822/3000 [10:15<00:14, 12.17it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  94%|█████████▍| 2825/3000 [10:15<00:12, 13.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  94%|█████████▍| 2827/3000 [10:15<00:12, 13.92it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  94%|█████████▍| 2829/3000 [10:16<00:23,  7.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  94%|█████████▍| 2833/3000 [10:16<00:21,  7.92it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  95%|█████████▍| 2838/3000 [10:16<00:15, 10.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  95%|█████████▍| 2840/3000 [10:17<00:22,  7.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  95%|█████████▍| 2844/3000 [10:17<00:18,  8.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  95%|█████████▍| 2848/3000 [10:18<00:15, 10.06it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  95%|█████████▌| 2851/3000 [10:19<00:25,  5.88it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  95%|█████████▌| 2856/3000 [10:19<00:15,  9.09it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  95%|█████████▌| 2863/3000 [10:19<00:08, 15.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  96%|█████████▌| 2866/3000 [10:20<00:18,  7.14it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  96%|█████████▌| 2870/3000 [10:21<00:15,  8.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  96%|█████████▌| 2876/3000 [10:21<00:09, 12.96it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  96%|█████████▌| 2880/3000 [10:21<00:08, 13.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  96%|█████████▌| 2882/3000 [10:21<00:09, 12.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  96%|█████████▌| 2884/3000 [10:22<00:12,  9.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  96%|█████████▋| 2888/3000 [10:22<00:13,  8.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  96%|█████████▋| 2892/3000 [10:23<00:10,  9.83it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  96%|█████████▋| 2894/3000 [10:23<00:09, 10.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  97%|█████████▋| 2896/3000 [10:23<00:10, 10.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  97%|█████████▋| 2898/3000 [10:23<00:10,  9.71it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  97%|█████████▋| 2900/3000 [10:23<00:11,  8.74it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  97%|█████████▋| 2902/3000 [10:24<00:09, 10.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  97%|█████████▋| 2906/3000 [10:24<00:08, 10.60it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  97%|█████████▋| 2908/3000 [10:24<00:08, 10.58it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  97%|█████████▋| 2912/3000 [10:25<00:07, 11.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  97%|█████████▋| 2918/3000 [10:25<00:06, 11.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  97%|█████████▋| 2923/3000 [10:25<00:05, 13.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  98%|█████████▊| 2927/3000 [10:26<00:08,  9.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  98%|█████████▊| 2930/3000 [10:26<00:06, 10.97it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  98%|█████████▊| 2936/3000 [10:27<00:04, 12.84it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  98%|█████████▊| 2938/3000 [10:27<00:04, 13.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  98%|█████████▊| 2940/3000 [10:27<00:05, 11.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  98%|█████████▊| 2942/3000 [10:27<00:04, 12.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  98%|█████████▊| 2946/3000 [10:28<00:06,  8.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  98%|█████████▊| 2948/3000 [10:28<00:05, 10.14it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  98%|█████████▊| 2954/3000 [10:28<00:03, 14.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  99%|█████████▊| 2957/3000 [10:29<00:03, 11.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  99%|█████████▊| 2959/3000 [10:29<00:04,  9.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  99%|█████████▊| 2961/3000 [10:30<00:06,  6.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  99%|█████████▉| 2963/3000 [10:30<00:05,  7.38it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  99%|█████████▉| 2969/3000 [10:30<00:02, 10.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  99%|█████████▉| 2973/3000 [10:31<00:03,  8.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  99%|█████████▉| 2975/3000 [10:31<00:02,  9.29it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  99%|█████████▉| 2978/3000 [10:31<00:01, 11.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  99%|█████████▉| 2983/3000 [10:32<00:01, 12.16it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow: 100%|█████████▉| 2985/3000 [10:32<00:01,  7.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow: 100%|█████████▉| 2987/3000 [10:33<00:02,  5.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow: 100%|█████████▉| 2989/3000 [10:33<00:01,  6.58it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow: 100%|█████████▉| 2995/3000 [11:02<00:12,  2.52s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow: 100%|█████████▉| 2999/3000 [11:02<00:01,  1.40s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow: 100%|██████████| 3000/3000 [11:02<00:00,  4.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "\u001b[32m2026-01-04 10:55:54.197\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36m__main__\u001b[0m:\u001b[36m<module>\u001b[0m:\u001b[36m4\u001b[0m - \u001b[1mEvaluation metrics (before optimization): {'f1': 0.0006702412868632708, 'em': 0.0006702412868632708, 'acc': 0.443029490616622}\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "logger.info(\"Evaluating workflow on test set...\")\n",
    "with suppress_logger_info():\n",
    "    results = textgrad_optimizer.evaluate(dataset=benchmark, eval_mode=\"test\")\n",
    "logger.info(f\"Evaluation metrics (before optimization): {results}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "1726fb5f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'f1': 0.0006702412868632708,\n",
       " 'em': 0.0006702412868632708,\n",
       " 'acc': 0.443029490616622}"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "e9fa0aec",
   "metadata": {},
   "outputs": [],
   "source": [
    "# qa_workflow = {\n",
    "#     \"goal\": \"Provide a direct answer to the question based on the context, without including explanations or reasoning.\",\n",
    "#     \"tasks\": [\n",
    "#         {\n",
    "#             \"name\": \"answer_generate\",\n",
    "#             \"description\": \"Generate a direct answer to the question based on the context.\",\n",
    "#             \"inputs\": [\n",
    "#                 {\"name\": \"question\", \"type\": \"str\", \"required\": True, \"description\": \"The question to answer directly.\"}\n",
    "#             ],\n",
    "#             \"outputs\": [\n",
    "#                 {\"name\": \"answer\", \"type\": \"str\", \"required\": True, \"description\": \"The direct answer to the question.\"}\n",
    "#             ],\n",
    "#             \"prompt_template\": StringTemplate(instruction=\"Think step by step to answer the question. You should explain your thinking process in the 'thought' field, and provide the final answer in the 'answer' field.\\nFormat your output in xml format, such as <thought>xxx</thought> and <answer>xxx</answer>.\"),\n",
    "#             \"parse_mode\": \"str\"\n",
    "#         }\n",
    "#     ] \n",
    "# }\n",
    "\n",
    "#generated_workflow 4omini\n",
    "qa_workflow = {\n",
    "    \"goal\": \"Provide a concise answer to the question using relevant context. The answer must be straightforward and avoid unnecessary explanations.\",\n",
    "    \"tasks\": [\n",
    "        {\n",
    "            \"name\": \"generate_answer\",\n",
    "            \"description\": \"Extract and formulate an answer from the given context.\",\n",
    "            \"inputs\": [\n",
    "                {\"name\": \"question\", \"type\": \"str\", \"required\": True, \"description\": \"The question that needs to be answered.\"},\n",
    "                {\"name\": \"context\", \"type\": \"str\", \"required\": True, \"description\": \"The background information pertinent to the question.\"}\n",
    "            ],\n",
    "            \"outputs\": [\n",
    "                {\"name\": \"answer\", \"type\": \"str\", \"required\": True, \"description\": \"The direct answer to the question.\"}\n",
    "            ],\n",
    "            \"prompt_template\": StringTemplate(instruction=\"Use the context to determine the best answer to the question. Provide your final answer in a clear format, without extra commentary or reasoning.\"),\n",
    "            \"parse_mode\": \"xml\"\n",
    "        }\n",
    "    ]\n",
    "}\n",
    "\n",
    "def collate_func(example: dict) -> dict:\n",
    "    context = \"\"\n",
    "    problem = \"Question: {}\\n\\nAnswer:\".format(example[\"question_new\"])\n",
    "    return {\"question\": problem, 'context':context}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "7434d744",
   "metadata": {},
   "outputs": [],
   "source": [
    "api_key = \"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\"\n",
    "# Function to encode the image\n",
    "import os\n",
    "os.environ[\"OPENAI_API_KEY\"] = api_key\n",
    "OPENAI_API_KEY = os.getenv(\"OPENAI_API_KEY\")\n",
    "\n",
    "\n",
    "llm_config = OpenAILLMConfig(model=\"gpt-4o-mini-2024-07-18\", openai_key=OPENAI_API_KEY, top_p=0.85, temperature=0.2, frequency_penalty=0.0, presence_penalty=0.0)\n",
    "llm = OpenAILLM(config=llm_config)\n",
    "executor_llm = OpenAILLM(config=llm_config)\n",
    "optimizer_llm = OpenAILLM(config=llm_config)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "ead42143",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2026-01-04 10:56:50.394\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.benchmark.pertqa\u001b[0m:\u001b[36m_load_data_from_file\u001b[0m:\u001b[36m52\u001b[0m - \u001b[1mloading HotPotQA data from /home/tl688/pitl688/selfevolve/EvoAgentX/examples/pertqa/reploge_train.json ...\u001b[0m\n",
      "\u001b[32m2026-01-04 10:56:50.473\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.benchmark.pertqa\u001b[0m:\u001b[36m_load_data_from_file\u001b[0m:\u001b[36m52\u001b[0m - \u001b[1mloading HotPotQA data from /home/tl688/pitl688/selfevolve/EvoAgentX/examples/pertqa/reploge_train.json ...\u001b[0m\n",
      "\u001b[32m2026-01-04 10:56:50.544\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.benchmark.pertqa\u001b[0m:\u001b[36m_load_data_from_file\u001b[0m:\u001b[36m52\u001b[0m - \u001b[1mloading HotPotQA data from /home/tl688/pitl688/selfevolve/EvoAgentX/examples/pertqa/reploge_test.json ...\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "benchmark = PertQA(pertdata='reploge')\n",
    "workflow_graph = SequentialWorkFlowGraph.from_dict(qa_workflow)\n",
    "agent_manager = AgentManager()\n",
    "agent_manager.add_agents_from_workflow(workflow_graph, executor_llm.config)\n",
    "\n",
    "evaluator = Evaluator(\n",
    "    llm=executor_llm, \n",
    "    agent_manager=agent_manager, \n",
    "    collate_func=collate_func, \n",
    "    num_workers=20, \n",
    "    verbose=True\n",
    ")\n",
    "\n",
    "textgrad_optimizer = TextGradOptimizer(\n",
    "    graph=workflow_graph, \n",
    "    optimize_mode=\"all\",\n",
    "    executor_llm=executor_llm, \n",
    "    optimizer_llm=optimizer_llm,\n",
    "    batch_size=3,\n",
    "    max_steps=20,\n",
    "    evaluator=evaluator,\n",
    "    eval_every_n_steps=1,\n",
    "    eval_rounds=1,\n",
    "    save_interval=None,\n",
    "    save_path=\"./\",\n",
    "    rollback=True,\n",
    "    constraints=[]\n",
    ")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "1e8e1560",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# len(benchmark._fulldata)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "5a9f95bc",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2026-01-04 10:56:54.315\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36m__main__\u001b[0m:\u001b[36m<module>\u001b[0m:\u001b[36m1\u001b[0m - \u001b[1mEvaluating workflow on test set...\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   0%|          | 13/3000 [00:01<03:21, 14.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   1%|          | 19/3000 [00:01<04:02, 12.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   1%|          | 28/3000 [00:02<02:51, 17.29it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   1%|          | 32/3000 [00:02<02:27, 20.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   1%|▏         | 41/3000 [00:02<02:50, 17.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   2%|▏         | 46/3000 [00:03<02:29, 19.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   2%|▏         | 50/3000 [00:03<02:35, 18.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   2%|▏         | 53/3000 [00:03<02:57, 16.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   2%|▏         | 56/3000 [00:03<03:20, 14.71it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   2%|▏         | 68/3000 [00:04<01:53, 25.72it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   2%|▏         | 72/3000 [00:04<02:06, 23.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   3%|▎         | 83/3000 [00:04<02:07, 22.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   3%|▎         | 89/3000 [00:05<01:44, 27.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   3%|▎         | 94/3000 [00:05<02:36, 18.61it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   3%|▎         | 103/3000 [00:05<02:04, 23.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   4%|▎         | 107/3000 [00:05<01:51, 25.89it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   4%|▎         | 111/3000 [00:06<02:28, 19.42it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   4%|▍         | 114/3000 [00:06<02:24, 20.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   4%|▍         | 122/3000 [00:06<02:10, 22.06it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metricsmetrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      " {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   4%|▍         | 128/3000 [00:07<02:27, 19.48it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   4%|▍         | 135/3000 [00:07<02:18, 20.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   5%|▍         | 143/3000 [00:07<02:03, 23.14it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   5%|▍         | 146/3000 [00:08<02:37, 18.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   5%|▌         | 155/3000 [00:08<02:09, 21.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   5%|▌         | 160/3000 [00:08<01:47, 26.49it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   5%|▌         | 164/3000 [00:08<02:30, 18.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   6%|▌         | 167/3000 [00:09<02:33, 18.40it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   6%|▌         | 174/3000 [00:09<02:31, 18.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531010d6ed0>\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   6%|▌         | 177/3000 [00:10<06:09,  7.65it/s]Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057094980>, 1555341.967904639)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531010d7190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531010d5650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530570978c0>, 1555344.008017087)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531010d5490>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153101a46690>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530577367b0>, 1555343.257340469)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153100e0f510>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153100da24d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153101081f10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530575f6890>, 1555341.813980507)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153100da3110>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153107809cc0>, 1555343.766840845)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153101083510>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153101b27510>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530575f5c50>, 1555345.024916029)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153101b27410>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530438d0610>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530577374d0>, 1555343.52274767)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530438f5850>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153043863350>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fcad7c40>, 1555346.225751384)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153043863210>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153101b0b3d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530438b0510>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057736dd0>, 1555343.161061428)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153101b0b1d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057ea60b0>, 1555345.85491345)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530438b0310>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530438c08d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302c78def0>, 1555345.141520504)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530438c1750>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531019e5890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd4e19b0>, 1555347.813816716)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153101abda90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531019b12d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153100d5d550>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057ea4e50>, 1555346.853319802)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530438d1150>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057ea57f0>, 1555346.247102942)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531019b1790>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153043828fd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057737930>, 1555347.902845386)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153043646d50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530438df750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057ea5390>, 1555347.999341291)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530438df790>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153043626f90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057735400>, 1555348.317470169)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153043893650>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530436eda10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd4e12b0>, 1555347.994421052)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153043634450>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530436fe7d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fcad6120>, 1555349.292951366)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530436fe250>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530436f7190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304284a7b0>, 1555349.769300109)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530436f7750>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153043892790>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304366cad0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd4e1be0>, 1555350.041316369)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304366e690>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fc917d20>, 1555348.723223659)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153043892690>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153101960910>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057c0fd20>, 1555350.142694923)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531019617d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304395e450>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057c0ef20>, 1555350.29402275)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304b0a5c10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304b0f5890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057d6a4a0>, 1555351.046668512)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304b0f5dd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304385d5d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057d682f0>, 1555351.234059755)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304b0f4110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304af64b50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056d6a580>, 1555351.527744345)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530439ec290>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304b04eb50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fcfc3460>, 1555352.429524516)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304ae14310>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304b04e750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fcfc04b0>, 1555352.171149435)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304b04cc50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304aea7ad0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057d6aba0>, 1555352.942898996)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304aea7dd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304ad367d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153106cd4910>, 1555353.722388603)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304ad35e90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304adb0950>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153106cd7cb0>, 1555353.737412692)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304adb0a90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304add9850>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fcfc0e50>, 1555353.977938833)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304addadd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304adeb950>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153107336820>, 1555354.768092214)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304adebb90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304ae69810>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304ada1610>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530429bb380>, 1555356.392773151)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304acfee50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153107335f60>, 1555355.079219201)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304ad26590>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304ae8dc90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531073359b0>, 1555354.707266452)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304393f3d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304abae050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530429bbd90>, 1555357.16178319)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304ae8dd10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304aabd4d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304aa857f0>, 1555358.920268731)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304aabe050>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304aa10210>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304aa842f0>, 1555359.015131945)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304aa11790>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304abe5bd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057d6a740>, 1555358.510861958)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304abe52d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304a919850>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530429bb850>, 1555358.661313163)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304a919990>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304a976fd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304a962a50>, 1555360.373227378)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304aa9e790>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304a81b810>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531073379a0>, 1555360.078048899)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304a81b6d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304a9dd590>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304a962820>, 1555360.084194063)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304a9ddc50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304a9c5ad0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304a80b550>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304a962660>, 1555360.182424661)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304a9c5a90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304a9638c0>, 1555360.983705266)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304a80af50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304a8492d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530429b8d70>, 1555360.264249556)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304abae650>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304a868210>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304a8a5e50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304a9a5610>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304a838d00>, 1555359.030203251)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304a868710>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304a839da0>, 1555360.155498067)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304a8a5d10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304a961390>, 1555360.152350442)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304a9a4590>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304aa2bc90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304ab94b40>, 1555391.78106758)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304aa9c590>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304a7c1bd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304a9f6f50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304a7f3410>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304a70edd0>, 1555391.661342139)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304acee9d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304a70de10>, 1555393.858421981)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304a796290>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304a7bc810>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304a614250>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304a70f620>, 1555391.762772031)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304a7bdf10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304a621110>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304a70fd20>, 1555391.782785525)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304a620f10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304a7b2650>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304a644d10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304a83ba80>, 1555391.515390419)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304a7b23d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304a648440>, 1555391.775420326)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304a644b50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304a7c3ed0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304a65b2d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304aa87150>, 1555391.789756305)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304a7c3f90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304a648ad0>, 1555393.658881796)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304a658910>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304a681f90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304a6a8390>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304a83b2a0>, 1555393.981916738)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304a659a50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304a649cc0>, 1555391.755883152)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304a6a8890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304a6c0d90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304a64a3c0>, 1555394.011386902)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304a6c0b90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304a683b10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304ab95400>, 1555393.876780957)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304a9b1250>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304a6cf6d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304a6ecc90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304a70e430>, 1555391.532846018)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304a7f3850>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304a70fa80>, 1555391.795689473)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304a613a50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304a64a970>, 1555394.150692537)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304a6cf590>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304a64aeb0>, 1555391.419701223)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304a6ecb50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304a6aa3d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304a50b4d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153101910590>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153100d4d4d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153101128510>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153101abf690>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153101aa4e10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153101acd2d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153101a6f490>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153101aa7650>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153101a51810>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153100e4f110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530438de3d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530438eb450>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304380a4d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531010f0390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304384e6d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304386eed0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304383dcd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153101aa6110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15310199f350>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531019606d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153101952b90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530438df210>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530438d1d90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153101a068d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153101a275d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304360ce10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531019fd490>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153101a21f50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153043644ed0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304369a750>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530436daa50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530439024d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153043935e90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304393dd10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530436f4b90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153101962c10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304368fa10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153043900710>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304397d890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153043982d50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153100e1a190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530436bb090>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304b00b590>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304b0344d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304b040710>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304397ec50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304b064b90>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304b078710>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304b08b2d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304b009190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304b0b1f10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304366d510>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304b0e0bd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530439e3090>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304af07c10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304af1fe50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304af7b5d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304af67d10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304af936d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304af4ac50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304afa93d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304b0b2210>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304afec310>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304aff7410>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304ae14bd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304ae2cd10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304ae43450>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304ae72fd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304afc7b90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304aed5050>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304aef1a50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304aefa650>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304ae6e3d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304af94890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304ad00a10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304af808d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304afed8d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304ad5b910>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304b06ded0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304ad39f10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304adbe6d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304adf6d10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304ac0f490>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304af94550>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304ac46f10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304ad52f50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304aeac250>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304ac7acd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304ac96150>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304acaab90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304acdcb50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304ad79890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304ab1c850>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304ad25490>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304ab2c150>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304ab43a50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304ac47290>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304acf7650>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304aea7a10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304abbaf50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304abd0a90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304abe4f10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304abf8e10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304aa1e850>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304aa34610>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304aa5c2d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304aa77610>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304aa2b090>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304aaa8b10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304aa123d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304aa8ead0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304abfb550>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304a944c10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304a94fc50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304a9be390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304a9a4750>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304a9b2550>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304a843150>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304a8c5650>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304a8d32d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304a8f8690>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304a8b0110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304a7061d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304a739bd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304a75ab90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304a720390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304a77da90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304a781d10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304a64b0e0>, 1555394.133423059)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304a8d08d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304a6482f0>, 1555392.389067744)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304a508b50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057094440>, 1555342.849370081)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531018c2dd0>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153100b77d90>, 1555342.294264993)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153100d4e490>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057095e80>, 1555343.025524453)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531010a91d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530575f5940>, 1555342.7867737)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153101abf150>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531057d1e10>, 1555343.216286532)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153101aa4cd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057734de0>, 1555343.715587472)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153101acd0d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057095390>, 1555342.950499717)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153101a6f350>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530575f5470>, 1555343.346821433)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153101aa7450>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057737460>, 1555343.485825256)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153101a52990>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057734b40>, 1555343.64717777)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153101a95a50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530575f7770>, 1555344.025489417)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530438de290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fcad7700>, 1555343.805718544)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530438eb310>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057735240>, 1555344.89498996)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304380a150>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057097310>, 1555344.622685609)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531010f0d10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fcad4980>, 1555344.39273367)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304384fe10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fcad4d00>, 1555344.893832683)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304386f5d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fcad6510>, 1555345.137211221)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153100fe04d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057735a90>, 1555344.906858922)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153101aa6b10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057ea4210>, 1555346.297544266)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15310199f210>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057ea69e0>, 1555346.791437897)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153101960bd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057ea5be0>, 1555345.694629192)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153101952a50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057ea5630>, 1555346.258187751)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530438de350>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd4e18d0>, 1555346.078048805)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304388a350>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd4e0600>, 1555346.588440882)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153101a06f90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd4e00c0>, 1555346.208689167)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153101a27490>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153100b77770>, 1555346.540761142)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304360ccd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd4e09f0>, 1555346.096566641)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531019fdad0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057ea5c50>, 1555347.88298182)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153101a223d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057ea55c0>, 1555347.313099569)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153043644b10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fc915d30>, 1555348.405968216)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304369a550>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd4e2970>, 1555347.653021409)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530436daf50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fc915010>, 1555347.950417152)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530439022d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057c0e120>, 1555348.048886708)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153043936890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057c0c980>, 1555348.154338636)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153043935890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057c0f1c0>, 1555348.502566273)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153043957b50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057ea6d60>, 1555347.796955191)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153101963e10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057c0cc20>, 1555348.135081258)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153100ffd050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057c0d2b0>, 1555348.548949817)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153043900350>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057c0e3c0>, 1555348.33248119)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304397df10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057ea49f0>, 1555349.546455966)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530439804d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fcad49f0>, 1555349.452867507)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153043808d90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057c0ee40>, 1555349.816482756)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530436ba5d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153042848130>, 1555349.702670575)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304b00b450>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fc9175b0>, 1555349.379692368)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304b034bd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153042848f30>, 1555350.505604384)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530436b8cd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304284b380>, 1555349.727232487)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304397ee50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530428482f0>, 1555349.759259922)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304b064a50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153042848830>, 1555349.96268349)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304b040cd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304284aac0>, 1555349.891732247)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304b06f190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fc9158d0>, 1555349.744392316)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304b009650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fc9148a0>, 1555349.691294962)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304b0b1d10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530428480c0>, 1555350.289146692)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153043975090>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057d69ef0>, 1555350.256384918)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304b0b5890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057d6b620>, 1555350.720713079)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530436fce50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fc9165f0>, 1555351.036975741)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304af07ad0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153042848c20>, 1555351.290880349)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304af1f8d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057c0f2a0>, 1555351.852211614)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304af7b490>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056d6bd20>, 1555351.70237793)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304b0b1c50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056d68600>, 1555351.144696416)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304af90dd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056d682f0>, 1555351.458949022)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304af4aed0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056d6bb60>, 1555351.743327551)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304afa91d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056d6a3c0>, 1555351.313223794)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304af57710>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057c0ce50>, 1555351.887761851)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304b0884d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153042848d70>, 1555352.001117654)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304aff4e10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153042849e80>, 1555352.311844099)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304b034c10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530575f78c0>, 1555351.996344134)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304ae2cbd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fcfc0980>, 1555352.273844881)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304ae43310>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fcfc20b0>, 1555352.687315169)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153101953850>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fcfc0de0>, 1555352.557601974)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304b0b2d10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fcfc0f30>, 1555352.566634647)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304aed54d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd5d8b40>, 1555353.217078134)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304aef1ed0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057735160>, 1555352.913439464)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304aefad10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fc914c20>, 1555352.760716096)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304ae6e290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fcfc3000>, 1555352.795255962)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304af94250>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fc916d60>, 1555353.131108869)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304ad00850>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153106cd4fa0>, 1555353.25630199)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304afbb1d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fcad4c90>, 1555353.70628537)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304afed910>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057ea5e80>, 1555353.921098034)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304ad59550>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fcfc3b60>, 1555355.371631439)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304ad3b990>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056d68210>, 1555354.032339051)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304ad39fd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fcfc23c0>, 1555353.944912858)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304adbd9d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153107334440>, 1555354.098222086)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304adf6b10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fcfc19b0>, 1555354.515576828)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304ac0f290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153107334670>, 1555354.221900381)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304af94490>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153106cd4670>, 1555354.401833061)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304ac44990>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153106cd50f0>, 1555354.471749301)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304ad52e10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153107336c10>, 1555354.595182386)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304aea7450>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304284ba10>, 1555354.815783236)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304ad8b210>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153107336ba0>, 1555355.3388659)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304ac96010>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153107337620>, 1555354.992488651)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304acab250>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530429b9780>, 1555355.453883214)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304acdca10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153106cd6510>, 1555355.451602202)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304ad799d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530429b8ec0>, 1555356.981120133)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304acdcd90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153106cd6890>, 1555355.730855228)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304ae14810>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056d6be00>, 1555355.83325987)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304ab2c610>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530429baeb0>, 1555355.477349772)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304af79390>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530429bbbd0>, 1555357.426957277)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304ac47810>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530429b84b0>, 1555355.570019472)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304acf7510>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153106cd4520>, 1555357.213351912)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304aea7210>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153107334d70>, 1555356.704761372)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304abbae10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304ab956a0>, 1555356.655291979)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304abd0890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304ab95a20>, 1555356.529220092)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304abe4d10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fcfc3bd0>, 1555356.489154973)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304af654d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153106cd4440>, 1555358.783662425)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304aa1d890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304ab96f20>, 1555357.147620047)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304ad909d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153042848210>, 1555358.675654304)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304aa34a10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153106cd7690>, 1555357.233591951)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304aa773d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304aa84590>, 1555357.32398617)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304aa2ba50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304ab974d0>, 1555357.300892184)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304aaab950>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304ab97b60>, 1555357.203965063)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304aa11f10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304aa84830>, 1555357.541547413)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304ab43a90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304ab95f60>, 1555358.098553392)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304abfb490>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304aa87460>, 1555358.256731911)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304a944ad0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304aa87e70>, 1555359.947481498)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304a94fb10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304a962040>, 1555359.150687101)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304a9bfb10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304aa84ad0>, 1555358.386420241)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304a9a4610>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530429b9550>, 1555359.170106611)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304a9b24d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304a961940>, 1555360.194756419)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304a843010>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304a83ac10>, 1555360.320066317)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304a8c5450>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530429bba80>, 1555360.307853165)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304a8d30d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304a70c280>, 1555361.206313856)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304a8f8550>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304aa863c0>, 1555394.157047865)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304a8b3510>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304a83b540>, 1555360.741132227)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304a705f10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304a961da0>, 1555361.320722996)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304a739810>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304a70d0f0>, 1555361.13333914)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304a75b250>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304a70d240>, 1555360.845614607)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304a720b10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304a70d8d0>, 1555360.992203307)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304a77d890>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304a70da20>, 1555361.266320509)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304a869dd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153101b09550>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304abac5d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057096c10>, 1555343.574324026)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153101b09410>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304ab94910>, 1555357.459627135)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304abac210>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304a7e9f50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057d68210>, 1555395.198636417)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304a546a10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304a77da50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd4e01a0>, 1555395.466613912)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304a668110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304a948290>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153100b74ec0>, 1555395.407965468)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304a949a10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304a5bcc10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153100b77f50>, 1555396.280785416)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304a5becd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304a8f9050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153100b74280>, 1555395.75085106)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304ac35890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304aed9510>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153105ce2cf0>, 1555396.469654585)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304aee54d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304b04f9d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153105ce0fa0>, 1555395.995654319)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304b04e110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530436c9dd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153105ce01a0>, 1555395.593457348)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530436ca610>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153101a6c5d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153105ce0d00>, 1555396.504866073)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530438e9710>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153101127ed0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304a5e6b10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304a8f85d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304a91b2d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304acb8fd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304ad8a810>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd4e02f0>, 1555395.970339171)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304a5e5e10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304a963460>, 1555395.362714045)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304a8f8990>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153100b76890>, 1555395.755575015)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304a91ab50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153100b75da0>, 1555397.531914811)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153101124e90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153100b74ad0>, 1555396.915540044)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304acb9350>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153100b774d0>, 1555395.248701414)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153103639790>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304a668150>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304a64a7b0>, 1555397.150267026)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304a6680d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304a400d10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bf1d630>, 1555397.282996038)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304afaa590>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304a558810>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153101af8c50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bf1faf0>, 1555397.576677269)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304a410990>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bf1f620>, 1555397.348698323)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153101af8c90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304a596e90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530436f64d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bf1f310>, 1555397.132595257)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304a596390>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fca3f150>, 1555397.610402148)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153100ddb510>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531010346d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153100e0d610>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153105ce1a90>, 1555397.519808332)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304a563610>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153101094490>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056e7a350>, 1555397.725287745)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153101095650>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153100d5d9d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153105ce11d0>, 1555398.852734415)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153100d5d810>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153100d68a90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304a72c290>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304a6461d0>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304a8254d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057c0e660>, 1555396.89841126)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304a76c4d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153105ce0de0>, 1555396.925424521)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304a824350>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304a529ad0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1532036546e0>, 1555397.068708497)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304a644f90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153100b74360>, 1555397.120054928)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304a52ac90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bf1f770>, 1555397.392314674)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153101034d10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531018c2b50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302c6fa430>, 1555399.21645333)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153100dba450>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153105ce37e0>, 1555399.111728649)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531018c0ed0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15310186ebd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304a5d0c50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304a70de80>, 1555400.443409289)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15310186f150>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153105ce1860>, 1555399.348209335)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304a5d3550>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304a563c10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531018b5650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153105ce0e50>, 1555398.721102724)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304a561b10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153101865b10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302c6f9390>, 1555398.590130139)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531018678d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531018f5f50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302c6fa200>, 1555398.837396152)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531018f7c50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153101898450>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153100fc8590>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302c6fab30>, 1555398.938716457)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153101899210>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302c6f8fa0>, 1555398.622339488)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153100fca410>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153100fd8dd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302c6f8c20>, 1555399.366785975)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153100fd8e50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153042fc4f50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302c6fb230>, 1555399.493391679)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153042fc5890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153042f6f890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302c6fb690>, 1555400.228226475)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153042f6d890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153100fe12d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303d354210>, 1555399.160194213)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531010975d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153042fecdd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303d357bd0>, 1555399.280212806)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153042fec950>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531010298d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153100f93410>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153100b77380>, 1555398.784932367)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531018b7f50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153042f82f10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302c6fb5b0>, 1555400.489072113)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153101028ad0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303d355e10>, 1555399.23817118)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153100f90050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304a83a3c0>, 1555400.442246078)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153042f81390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15310184c550>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fca3faf0>, 1555399.538843469)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15310184d450>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153101709d50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153100f22890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bf1eeb0>, 1555400.524457491)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153101709990>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b558de0>, 1555400.245887345)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153100f22790>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153100e69190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b55a580>, 1555400.464419483)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153100e69d50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153100e69810>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153100f1a010>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153100b76120>, 1555400.858724202)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153100e694d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304a648830>, 1555400.571869423)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153100f19b90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153100ea1450>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b558590>, 1555400.274784567)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153100ea1590>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531018d6850>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bf1dbe0>, 1555400.482424317)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153042f24e50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153042a26d50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153100e7b2d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303d356c10>, 1555400.699890913)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153100e4d890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b55b070>, 1555402.263981736)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153100e7b710>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153101664b10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153042ad7090>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304a64bee0>, 1555400.831281824)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153101695710>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b55add0>, 1555400.949882907)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153042ad6790>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153101639b90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153105ce38c0>, 1555400.603358631)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15310163a510>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531016f0450>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153100e9c4d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048b25400>, 1555400.902357826)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153100e0c050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048b25390>, 1555402.337173962)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153100e9c490>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153100f07310>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153042af7410>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304a70fee0>, 1555402.405270218)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153100f0df50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048b27930>, 1555401.64950576)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153042af7e90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153042a41bd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048b27d20>, 1555401.633351186)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153042a418d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304af1e8d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153042bae990>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048b27150>, 1555402.379354926)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153042a95510>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048b27230>, 1555402.005991346)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153042bacbd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153042a2a510>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153042a61b10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303d355cc0>, 1555401.825866836)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153042a2a590>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531043d2040>, 1555402.175314359)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153042aa54d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153042a9fb90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153042b70a50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303d357620>, 1555402.561174411)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153042a9db90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531043d0750>, 1555402.556565365)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153042b72110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153100f5f9d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153100eabc50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153105ce2270>, 1555403.036700662)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153100f5f810>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531043d0d70>, 1555402.374847553)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153042a004d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153042bb8310>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531043d1c50>, 1555403.131768126)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153042bbadd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153100e696d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531043d35b0>, 1555403.465672532)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153042f27d90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153101865bd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153100eff8d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153042b7dd50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153100eca050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531043d2ba0>, 1555403.798354503)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153100ebb950>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303fce75b0>, 1555403.776127044)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153100effe90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531043d0c90>, 1555403.22210608)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153042b7db10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304aa85630>, 1555404.796418795)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153100ecbc10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153042fecb50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153042dde450>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153105245240>, 1555403.608596127)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153100e4c4d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153105247cb0>, 1555403.817800539)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153042ddf190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153042bafb50>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153042e8d850>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302c6f8de0>, 1555403.90855157)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153042badd10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b55a3c0>, 1555403.749641353)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153042ada290>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153042e37b50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531043d1010>, 1555403.97019344)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153042e8e9d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153042e6eed0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b8555c0>, 1555404.65863074)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153042e6de90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153042da17d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153042edc810>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303fce6820>, 1555404.612404633)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153042d6afd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b857150>, 1555404.192373335)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153042edcad0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153101915ad0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153100eaad50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b855a90>, 1555404.305590032)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153042eef950>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b857a10>, 1555404.914542247)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153100eab050>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153042a41210>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b857850>, 1555404.520422504)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153042a9e590>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530430eed90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530430afad0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153042de8b10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153042d414d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303fce74d0>, 1555404.026220886)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153042de8450>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048b25470>, 1555404.754595461)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153042d382d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b855a20>, 1555404.9619083)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530430afc10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153105245d30>, 1555403.558713873)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153042d438d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153100fdb150>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b855be0>, 1555405.356582369)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153101913150>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153105ba37d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b558750>, 1555405.469626264)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153105ba3bd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153105d1b850>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302c83d400>, 1555406.642677092)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153042da1750>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530430e1f90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd758c90>, 1555405.040349947)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530430e1150>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153042dac450>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153105618f90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302c83eb30>, 1555405.415743003)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153100e9cdd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd75baf0>, 1555405.58049798)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531055d4d10>\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metricsmetrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      " {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153105bd1890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530430cda50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15310244abd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd7598d0>, 1555405.562928342)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153042d40450>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302c83cd70>, 1555406.703873664)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531024495d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531055b64d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056c51da0>, 1555406.766209164)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531055b6d50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153105e017d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153106372fd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b559cc0>, 1555408.300432064)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153105e01cd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bf1c750>, 1555406.185926153)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153106370310>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153102cadd90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048b26580>, 1555406.378190476)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153102cada10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15310267d890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b854de0>, 1555406.251497733)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15310267e090>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153102adb510>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd75ab30>, 1555405.619337216)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15310611e750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b8540c0>, 1555408.21210222)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153102ada810>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153102250690>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153102102a90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303fcd5ef0>, 1555406.913025384)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153102251810>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fc0a1320>, 1555406.716156917)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153102102690>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153102085d10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd13d2b0>, 1555406.961099455)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153102085190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153105ba19d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057fde580>, 1555408.412590004)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531059dfc90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302c71d110>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302cadf7e0>, 1555408.463112699)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302c71d710>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302c771450>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fcee09f0>, 1555408.165294687)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304b139990>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531040ee6d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153042dafd90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153042b91750>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531027e01d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302c7aa4d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153101da7790>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302c721710>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303fcd5be0>, 1555406.329090966)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531027e3cd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153103d06c10>, 1555408.084104247)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153100f5de10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303ef02a50>, 1555406.573829579)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153042dae2d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056ddbaf0>, 1555406.495749138)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531052e0b50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd506dd0>, 1555407.986832226)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153106221210>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057fdcbb0>, 1555408.82333738)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153101da7a90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057fdd160>, 1555408.779704712)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302c720450>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302c78a510>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057fdce50>, 1555409.211479174)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531003a4e50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153100bb7b10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153103d07a10>, 1555408.337609939)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153100bb4910>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531027e3050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303fce7cb0>, 1555408.450096974)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531027e1cd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531028fe6d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fcc7d860>, 1555408.875755866)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302c700c10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153102062bd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530429b9320>, 1555409.710387588)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531078ae0d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153102206d90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b65b770>, 1555410.206160788)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531055b4dd0>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153106d966d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302cadf620>, 1555409.592873328)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153106d96650>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531076056d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153101872890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153101bee210>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531063d7950>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531057ffd90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153105cff5d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531078a9210>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fcc7d1d0>, 1555409.342143299)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15310783dcd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fcc7cd00>, 1555409.773219977)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531057fd890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fcee1080>, 1555408.224013535)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153101beda50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530429b8360>, 1555410.36835685)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153105cfc390>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b65a740>, 1555410.223839488)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303dd85710>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531043d0b40>, 1555408.040006256)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304b1fb950>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fcc7c6e0>, 1555408.457583464)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531063d7590>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15310580d890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fcc7fd90>, 1555410.295197737)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15310580d2d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153107654050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056c539a0>, 1555410.18086405)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531041104d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153106500390>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153056c53bd0>, 1555410.36263041)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153106503450>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153101b63910>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b65b1c0>, 1555410.343162587)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153055ab9f50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531075788d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153106ab7010>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd220e50>, 1555411.117724229)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153107578e50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd222f20>, 1555411.208143801)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153106ab6910>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153107365810>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd2206e0>, 1555410.501261785)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302c7cfd10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531073546d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153107545510>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fcc7e580>, 1555411.071685825)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153107357a50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd223230>, 1555410.602478741)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153107546b90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530ffe3b550>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd2210f0>, 1555411.376632889)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530ffe3bdd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531077f7490>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fcc7e5f0>, 1555411.455204917)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531077f5c50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153106fd4090>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd220440>, 1555411.509277921)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153106fd5990>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303cf231d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd222eb0>, 1555411.49924284)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fce6a490>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531021bbe90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303dc396d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153105d3fc90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303dda1250>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153101bec150>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302cadcde0>, 1555409.978441516)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153101bef1d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530429bb070>, 1555410.042930405)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303dda3850>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303f1efc40>, 1555411.354635201)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153107392450>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530428488a0>, 1555412.200566201)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303dc38d50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302cadc590>, 1555409.764850846)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153105d3d150>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531076557d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057fdc9f0>, 1555412.078606188)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153105d684d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531069cc610>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd8d58d0>, 1555412.390336265)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531069cd290>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15310700bed0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153106b17650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd8d5390>, 1555412.26539336)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15310700ba10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd8d46e0>, 1555412.229139889)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153106b16350>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15310763dd50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd8d4750>, 1555411.903067194)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fe3ed4d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303da01e10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153057940610>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b658440>, 1555412.071645088)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303da00210>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302ca38210>, 1555412.48564361)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153057941910>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153107391ad0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531061cc810>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b659be0>, 1555412.473808249)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153107391e50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d3c3c90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302ca39c50>, 1555412.433136249)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fc408450>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fc132410>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15310221ce90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302ca39f60>, 1555412.899254259)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15310221f610>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d391490>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302ca3a660>, 1555412.908532271)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d391dd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153107698310>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531072c7cd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd222ac0>, 1555412.521459609)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531061cef10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd2216a0>, 1555412.953338964)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fd4d4f10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd222580>, 1555413.234519893)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153107699350>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd8d54e0>, 1555411.646521532)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304b1c9610>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153106663c90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303e2d0c90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd220280>, 1555413.35460929)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530579ce050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057736ac0>, 1555413.273123969)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303e2d1d10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530566132d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531075908d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303d0397f0>, 1555413.175119572)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531075929d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fd413cd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd8d62e0>, 1555413.65401062)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fc7d3610>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303db66b50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303d03a510>, 1555413.588587476)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303db66410>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303d7f9dd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153105221210>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd8d4c20>, 1555414.346933664)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303d7fb0d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303d039080>, 1555414.17298582)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153105222950>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531051c8810>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303e22e5d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303fce7310>, 1555414.14739047)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531064e2ed0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153102888d90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303dc5ba90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057094360>, 1555414.098430523)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fd750710>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b658a60>, 1555413.894225274)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153055c27e50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15305559f8d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057094600>, 1555414.79265671)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15305559c290>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303d3706d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303da63b50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530570971c0>, 1555414.890608016)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303d818d10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fcc7d080>, 1555414.372411366)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303da34510>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303d767310>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd223a80>, 1555413.61513828)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531074816d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303d03b000>, 1555413.835205122)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153107607b90>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530575f5a90>, 1555414.663055004)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303d9a3790>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303f2ded50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530575f5780>, 1555415.130335968)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303f2dd210>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303d11a190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302ca38de0>, 1555415.365012263)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303d1188d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153055300590>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302c78e580>, 1555414.990031507)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153048d4e510>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303d325b10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd8d77e0>, 1555415.140919346)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303d326750>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303d7c8610>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd2227b0>, 1555415.369314714)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303d7cbc50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303f236990>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153034d82850>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302c78e5f0>, 1555415.739107746)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303f4d2410>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302c78d940>, 1555415.261976776)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153034c93c10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303e643210>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153057094670>, 1555415.559940794)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303cfaafd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303e024c10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153104018210>, 1555415.657100135)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153055802690>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303d2c5a50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153104019240>, 1555415.642633128)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303d2c51d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303d25ef50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153035034f50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531040192b0>, 1555416.380809338)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fc13add0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303d039320>, 1555415.893632534)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153056fca990>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153055a4f550>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304b8d07d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153104018c20>, 1555415.755741055)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303e5565d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303faa5950>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530575f7620>, 1555414.534883808)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303cf84650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530575f5b70>, 1555414.479921293)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153106adb410>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530359fc050>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303d7d9110>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15310401a0b0>, 1555416.297938263)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530359fdf90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531040190f0>, 1555416.470639089)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303d7db7d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302dbe4e10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531040188a0>, 1555416.254461482)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302dbe6790>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153055467950>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153104019b70>, 1555416.374081611)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153055464cd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fc20dc10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15310401ae40>, 1555416.83601956)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153042f4d890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302c80cdd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531057d07c0>, 1555417.599284465)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302c80f110>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303537b250>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303d034b50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531057d1390>, 1555416.773037127)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153055a84bd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153106fb5e90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531057d3070>, 1555417.743594468)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302da05f90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530355f0d90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302c8b1790>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531057d0440>, 1555417.285408541)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530355f1a90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531057d1240>, 1555417.274663893)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530428f6150>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302da29f10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531057d32a0>, 1555417.259066055)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153049b01710>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153034cbf790>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531057d24a0>, 1555416.909565892)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153034cbd490>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302db74f50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302c78f8c0>, 1555417.333070163)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302db77f10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302cae1c90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302c87e390>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd8d4980>, 1555416.71300601)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303d035f90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531038d7230>, 1555417.348219425)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302cae1710>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531038d6740>, 1555417.977419649)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302c87f1d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302cbb3ed0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531038d4de0>, 1555417.828708476)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302cbb3850>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302c98dc50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153034c52310>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302c78f460>, 1555419.342787584)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302c98c4d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531038d6580>, 1555417.761966033)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153034c52ad0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d3ded10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531038d4210>, 1555418.388444885)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d3de410>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302daf4dd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d55fdd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531038d6040>, 1555419.208846916)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302dbe7b10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531078096a0>, 1555419.320137242)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d55e510>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302c8d8790>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302c8e7550>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15302ca3ad60>, 1555418.955259016)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302c8e5890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302cbb8c90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d382a90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15310780a5f0>, 1555419.045038261)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d3166d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531038d72a0>, 1555418.665935616)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d3828d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302ccae6d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304b65a270>, 1555419.14477118)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302ccad290>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d3d6c50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153107809da0>, 1555418.821771411)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fc97ee90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303fcefd10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15303413d9d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531038d42f0>, 1555418.582231373)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302c7fed90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531038d60b0>, 1555417.342790363)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303413e390>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531038d73f0>, 1555417.390204484)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302ca82510>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15310780b000>, 1555420.539406254)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302cc41a50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302cbaa190>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531028f8050>, 1555419.200843997)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d5e84d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d317ed0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531028f9780>, 1555420.300641708)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530ffc4b250>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d347490>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531028f9a20>, 1555420.849030757)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d347510>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302da94390>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531028f8520>, 1555421.957343492)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302da95ad0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d6bd210>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153107809320>, 1555420.610793172)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d6be150>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d9abbd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153105036eb0>, 1555420.685510028)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d9aae50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d85fc90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531050364a0>, 1555420.74449205)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d5cf190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d730a90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153105035e10>, 1555420.645420031)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d732990>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153101fdfa90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302cc0d3d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302cb7e450>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302dbd2050>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302c98e050>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d577850>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531057d1a20>, 1555421.780912917)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d9d3890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531028f88a0>, 1555419.260894205)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302cc0f150>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531028faa50>, 1555418.928047633)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302cb7f0d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531028fab30>, 1555418.932347573)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302dbd3890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531028f9160>, 1555420.538216137)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530341ab090>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531028f8670>, 1555420.316517156)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d50d890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15310533d1d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fce777e0>, 1555452.019687414)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15310533ec90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531074a9c50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fce752b0>, 1555454.284923674)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531074a9990>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15310637de50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153106e10650>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fce76f90>, 1555454.420973475)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15310637d910>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fce75b00>, 1555421.893928508)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fcd52b90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530ffa82e90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153106ad5e50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fce74b40>, 1555454.362450323)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530ffa802d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531028fa900>, 1555451.80736086)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153102bff950>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153100b950d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531028f9d30>, 1555451.71125003)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153102b88150>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153101e9c690>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531024b4b50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302ca81790>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302c8cb090>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531078081a0>, 1555421.797702933)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153101e9c950>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530fd9cd610>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fce76b30>, 1555422.029071963)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302c8cbbd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fce772a0>, 1555422.045325267)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153105c46f90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302c8c9910>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fce75d30>, 1555420.844851493)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153101f34890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd1db150>, 1555420.803803916)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d4270d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304be56090>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153107809b00>, 1555420.945865872)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302da8bb50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fce77620>, 1555454.519108278)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153100b96890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fce766d0>, 1555420.907916086)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302db15890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304293d4d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531038d6a50>, 1555455.878032201)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153042930190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531074aa850>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530ffba7890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531022f7910>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531021486d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302cbfdb50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153106f57ad0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530429c82d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153106292090>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531061d5410>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153106572fd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530429a5e90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304313da50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531050349f0>, 1555454.366721719)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531074aaa50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd1d92b0>, 1555451.892783025)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fdc6e710>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153106407d90>, 1555454.548216884)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531022f74d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153106407380>, 1555451.869971552)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153106426090>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153106404600>, 1555451.934458424)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302cbfd2d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531028fa430>, 1555451.609430769)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303cf15890>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153106406740>, 1555451.605207075)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530429c9350>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fce77930>, 1555455.558005876)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153106291410>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd18c980>, 1555455.976668123)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531061d6050>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531057d00c0>, 1555455.466189872)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153106570750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd18f540>, 1555456.803983412)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d84fe90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd18d080>, 1555455.848443062)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304313d690>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302d7f5cd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304a54e0b0>, 1555454.374959757)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530429f1510>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304297b5d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153042cf84d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bcb23c0>, 1555457.238352174)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153042cfa010>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153042cd5b90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15310655f550>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153101cc8890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530429236d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531073f8610>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153042912c90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530431cca10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531038d4ad0>, 1555455.812261307)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304297a710>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153106404360>, 1555455.778688009)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153042923b50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304a54cf30>, 1555455.855170968)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531073f97d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304a54f150>, 1555455.646663008)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153042911310>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304a54dc50>, 1555454.39098385)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530429f84d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd18f2a0>, 1555455.995833023)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302d8a2590>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530429bed50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530ffbdf540>, 1555457.17061563)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531064d21d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304a54db00>, 1555454.269188655)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530fdc296d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bcb26d0>, 1555457.518799872)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15302dac54d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15302dad2890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304a54c520>, 1555453.74938044)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531024796d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153043535410>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530ffbdd390>, 1555457.46899076)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15303fb46010>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153043531590>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153107808ad0>, 1555456.928830397)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153043531790>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153042995250>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd18c830>, 1555457.239526307)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153042994210>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530435e0550>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304a54ec80>, 1555457.438224775)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1530435e3f50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153106ea5810>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531064064a0>, 1555457.437181991)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153106ea7910>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153042cc4c90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303d140130>, 1555457.336846373)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153042cc5a50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153106e14b10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fce74130>, 1555457.504675598)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153106e14890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153042c1dc50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153106dca510>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304a54c980>, 1555459.466439365)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153042c1ec50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fd1da3c0>, 1555457.363982135)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153106ddd890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153106db5ed0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153106f5ce10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303d1417f0>, 1555459.094447159)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153106fa5890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530fce75da0>, 1555457.89565274)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153106f5da90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153106e65090>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303d142200>, 1555457.571807668)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153106e67410>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153101ceb650>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153101753d90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304352a7d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530ffbddbe0>, 1555457.463497963)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153101cea010>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530ffbdeac0>, 1555457.069205763)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153101753750>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304a54d470>, 1555457.081360098)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153043528cd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153106e47010>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530ffbdf4d0>, 1555459.050629221)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153106e47fd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304294dc10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15304295ef10>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304a54ef20>, 1555459.542437781)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304294e410>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530ffbdf1c0>, 1555458.712239129)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15304295e450>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153106dca790>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153101753c50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303d143070>, 1555459.179131504)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153106dca550>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303d140600>, 1555458.845085301)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531017504d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153106eb54d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153107616430>, 1555458.836333495)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153043112790>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1530559ec710>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153043530e50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530ffbdeb30>, 1555459.085115464)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153106c0bdd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303d1437e0>, 1555459.112604921)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153043532610>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153105e6df90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153107617a10>, 1555459.141080938)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153105e6e390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153106c71990>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153105e32150>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153107617af0>, 1555460.783803146)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153106c72d50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153107617150>, 1555459.341172229)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153105e32010>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153106eccd90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153105f80890>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531076171c0>, 1555459.333469036)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153042c669d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531064074d0>, 1555459.255101189)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153105f80c10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153106f7cc90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153105f4e4d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153107615630>, 1555459.527192294)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153106f7d3d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153107615080>, 1555459.608891968)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153105f4e2d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153105ee2c90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153107615780>, 1555459.811535243)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153105ee3490>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153105eca550>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530489747c0>, 1555459.782500293)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153105e32dd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153105e81450>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15304bcb0440>, 1555460.729180522)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153105e80650>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153106900a50>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048975080>, 1555460.542297606)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153106900ad0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531068d6fd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048976350>, 1555461.327319603)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531068d6bd0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153105e13950>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153048977f50>, 1555460.31927319)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153105e13d90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153106c94410>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1531064050f0>, 1555460.55763682)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153105e6f490>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153102d13390>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153102c90810>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530488ba040>, 1555460.972962002)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153102d13610>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530488b8fa0>, 1555460.874058356)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153102c93890>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531068f9d10>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531077aacd0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153107617ee0>, 1555460.710946169)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531068fbe50>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530488b8d70>, 1555460.947903835)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531077ab650>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153102cdc690>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530488b99b0>, 1555460.923635533)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153102c362d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531070df610>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530488b9400>, 1555461.128557475)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531070df9d0>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153102cf7a50>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x15310770b7d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530488b86e0>, 1555461.171986839)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153102cf6690>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x15303d141e10>, 1555462.415507412)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153107708350>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153107718390>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x1530488b83d0>, 1555461.279098217)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15310771ba90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x1531071a1190>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153107181ed0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153042c4b0e0>, 1555462.429609762)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x1531071a2f90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153107614980>, 1555462.457515586)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153107183350>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153107159d90>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153042c4aa50>, 1555461.979413195)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x153107158d90>\n",
      "Unclosed client session\n",
      "client_session: <aiohttp.client.ClientSession object at 0x153105eca5d0>\n",
      "Unclosed connector\n",
      "connections: ['deque([(<aiohttp.client_proto.ResponseHandler object at 0x153042c4a510>, 1555462.433654)])']\n",
      "connector: <aiohttp.connector.TCPConnector object at 0x15310772f090>\n",
      "Evaluating workflow:   6%|▌         | 185/3000 [00:11<04:42,  9.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   7%|▋         | 198/3000 [00:11<03:00, 15.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   7%|▋         | 201/3000 [00:11<02:45, 16.93it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   7%|▋         | 208/3000 [00:12<02:47, 16.66it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   7%|▋         | 217/3000 [00:12<02:21, 19.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   7%|▋         | 223/3000 [00:13<02:49, 16.37it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   8%|▊         | 229/3000 [00:13<02:26, 18.93it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   8%|▊         | 233/3000 [00:13<02:20, 19.63it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   8%|▊         | 239/3000 [00:13<02:15, 20.35it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   8%|▊         | 247/3000 [00:14<02:29, 18.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   8%|▊         | 253/3000 [00:14<02:24, 19.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   9%|▊         | 257/3000 [00:14<02:11, 20.83it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   9%|▊         | 262/3000 [00:15<02:36, 17.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   9%|▉         | 268/3000 [00:15<02:30, 18.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:   9%|▉         | 275/3000 [00:15<02:01, 22.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:   9%|▉         | 279/3000 [00:15<01:47, 25.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  10%|▉         | 285/3000 [00:16<02:33, 17.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  10%|▉         | 289/3000 [00:16<02:16, 19.86it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  10%|▉         | 292/3000 [00:16<02:34, 17.48it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  10%|▉         | 299/3000 [00:16<01:46, 25.42it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  10%|█         | 302/3000 [00:17<02:25, 18.48it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  10%|█         | 308/3000 [00:17<02:25, 18.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  10%|█         | 313/3000 [00:17<02:32, 17.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  10%|█         | 315/3000 [00:17<02:44, 16.32it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  11%|█         | 323/3000 [00:18<02:35, 17.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  11%|█         | 327/3000 [00:18<02:10, 20.47it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  11%|█         | 333/3000 [00:18<02:40, 16.58it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  11%|█         | 335/3000 [00:19<02:53, 15.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  11%|█▏        | 339/3000 [00:19<03:12, 13.82it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  12%|█▏        | 349/3000 [00:20<02:37, 16.80it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  12%|█▏        | 352/3000 [00:20<02:44, 16.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  12%|█▏        | 354/3000 [00:20<02:47, 15.83it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  12%|█▏        | 362/3000 [00:20<02:16, 19.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  12%|█▏        | 365/3000 [00:20<02:35, 16.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  12%|█▏        | 372/3000 [00:21<02:13, 19.71it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  12%|█▎        | 375/3000 [00:21<02:33, 17.14it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  13%|█▎        | 379/3000 [00:21<02:34, 17.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  13%|█▎        | 385/3000 [00:22<02:34, 16.96it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  13%|█▎        | 391/3000 [00:22<02:28, 17.63it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  13%|█▎        | 393/3000 [00:22<02:49, 15.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  13%|█▎        | 396/3000 [00:22<02:54, 14.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  13%|█▎        | 400/3000 [00:23<02:20, 18.55it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  13%|█▎        | 403/3000 [00:23<02:38, 16.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  14%|█▎        | 408/3000 [00:23<02:56, 14.66it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  14%|█▎        | 411/3000 [00:23<02:41, 16.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  14%|█▍        | 419/3000 [00:24<01:51, 23.07it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  14%|█▍        | 423/3000 [00:24<02:12, 19.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  14%|█▍        | 426/3000 [00:24<02:24, 17.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  14%|█▍        | 431/3000 [00:24<02:51, 15.02it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  14%|█▍        | 435/3000 [00:25<02:15, 18.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  15%|█▍        | 441/3000 [00:25<01:59, 21.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  15%|█▍        | 446/3000 [00:25<02:59, 14.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  15%|█▌        | 451/3000 [00:26<02:33, 16.58it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  15%|█▌        | 454/3000 [00:26<02:11, 19.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  15%|█▌        | 457/3000 [00:26<02:25, 17.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  15%|█▌        | 462/3000 [00:26<03:09, 13.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  16%|█▌        | 465/3000 [00:27<02:40, 15.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  16%|█▌        | 471/3000 [00:27<02:16, 18.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  16%|█▌        | 474/3000 [00:27<02:12, 19.04it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  16%|█▌        | 480/3000 [00:27<02:20, 17.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  16%|█▌        | 484/3000 [00:28<02:57, 14.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  16%|█▋        | 492/3000 [00:28<01:46, 23.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  17%|█▋        | 498/3000 [00:28<02:49, 14.78it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  17%|█▋        | 502/3000 [00:29<02:38, 15.71it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  17%|█▋        | 510/3000 [00:29<01:48, 22.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  17%|█▋        | 513/3000 [00:29<02:08, 19.40it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  17%|█▋        | 519/3000 [00:30<02:32, 16.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  18%|█▊        | 526/3000 [00:30<01:54, 21.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  18%|█▊        | 529/3000 [00:30<02:00, 20.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  18%|█▊        | 532/3000 [00:30<02:17, 17.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  18%|█▊        | 538/3000 [00:31<02:34, 15.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  18%|█▊        | 542/3000 [00:31<02:04, 19.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  18%|█▊        | 548/3000 [00:31<02:23, 17.09it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  18%|█▊        | 550/3000 [00:31<02:37, 15.52it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  19%|█▊        | 557/3000 [00:32<01:49, 22.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  19%|█▉        | 563/3000 [00:32<02:08, 18.96it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  19%|█▉        | 566/3000 [00:32<02:53, 14.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  19%|█▉        | 568/3000 [00:33<03:00, 13.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  19%|█▉        | 574/3000 [00:33<02:22, 17.04it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  19%|█▉        | 578/3000 [00:33<02:30, 16.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  19%|█▉        | 580/3000 [00:33<03:19, 12.16it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  19%|█▉        | 584/3000 [00:33<02:39, 15.11it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  20%|█▉        | 589/3000 [00:34<02:23, 16.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  20%|█▉        | 595/3000 [00:34<01:43, 23.27it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  20%|█▉        | 598/3000 [00:34<02:16, 17.58it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  20%|██        | 602/3000 [00:35<02:43, 14.63it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  20%|██        | 608/3000 [00:35<02:19, 17.17it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  20%|██        | 614/3000 [00:35<02:23, 16.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  21%|██        | 618/3000 [00:35<02:22, 16.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  21%|██        | 620/3000 [00:36<02:22, 16.72it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  21%|██        | 625/3000 [00:36<02:24, 16.46it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  21%|██        | 627/3000 [00:36<02:39, 14.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  21%|██        | 633/3000 [00:37<02:40, 14.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  21%|██▏       | 639/3000 [00:37<01:56, 20.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  21%|██▏       | 644/3000 [00:37<02:26, 16.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  22%|██▏       | 648/3000 [00:37<02:41, 14.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  22%|██▏       | 652/3000 [00:38<02:20, 16.66it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  22%|██▏       | 659/3000 [00:38<01:54, 20.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  22%|██▏       | 662/3000 [00:38<01:46, 22.06it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  22%|██▏       | 665/3000 [00:38<03:03, 12.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  22%|██▏       | 668/3000 [00:39<03:00, 12.89it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  23%|██▎       | 676/3000 [00:39<02:00, 19.29it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  23%|██▎       | 679/3000 [00:39<02:15, 17.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  23%|██▎       | 682/3000 [00:39<02:36, 14.82it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  23%|██▎       | 690/3000 [00:40<01:46, 21.78it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  23%|██▎       | 693/3000 [00:40<02:01, 19.05it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  23%|██▎       | 696/3000 [00:40<02:01, 18.97it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  23%|██▎       | 701/3000 [00:41<02:33, 14.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  24%|██▎       | 705/3000 [00:41<01:59, 19.17it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  24%|██▎       | 708/3000 [00:41<02:22, 16.14it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  24%|██▍       | 714/3000 [00:41<02:15, 16.89it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  24%|██▍       | 720/3000 [00:42<02:11, 17.36it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  24%|██▍       | 722/3000 [00:42<02:47, 13.63it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  24%|██▍       | 727/3000 [00:42<02:15, 16.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  24%|██▍       | 730/3000 [00:42<02:06, 17.96it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  24%|██▍       | 735/3000 [00:43<02:29, 15.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  25%|██▍       | 738/3000 [00:43<02:08, 17.57it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  25%|██▍       | 741/3000 [00:43<02:31, 14.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  25%|██▍       | 745/3000 [00:43<02:00, 18.71it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  25%|██▌       | 751/3000 [00:44<02:33, 14.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  25%|██▌       | 754/3000 [00:44<02:47, 13.38it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  25%|██▌       | 758/3000 [00:44<02:34, 14.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  25%|██▌       | 762/3000 [00:45<02:41, 13.87it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  25%|██▌       | 764/3000 [00:45<02:47, 13.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  26%|██▌       | 771/3000 [00:45<02:20, 15.84it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  26%|██▌       | 773/3000 [00:45<02:37, 14.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  26%|██▌       | 777/3000 [00:46<02:35, 14.29it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  26%|██▌       | 782/3000 [00:46<02:09, 17.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  26%|██▌       | 784/3000 [00:46<02:21, 15.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  26%|██▋       | 789/3000 [00:46<02:39, 13.88it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  26%|██▋       | 794/3000 [00:47<02:26, 15.04it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  27%|██▋       | 798/3000 [00:47<01:58, 18.60it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  27%|██▋       | 803/3000 [00:47<02:38, 13.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  27%|██▋       | 808/3000 [00:48<02:15, 16.17it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  27%|██▋       | 814/3000 [00:48<01:54, 19.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  27%|██▋       | 821/3000 [00:48<02:09, 16.84it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  27%|██▋       | 824/3000 [00:49<02:43, 13.32it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  28%|██▊       | 827/3000 [00:49<02:29, 14.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  28%|██▊       | 836/3000 [00:49<02:05, 17.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  28%|██▊       | 843/3000 [00:50<02:16, 15.78it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  28%|██▊       | 848/3000 [00:50<02:21, 15.17it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  28%|██▊       | 850/3000 [00:51<02:48, 12.77it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  29%|██▊       | 859/3000 [00:51<02:05, 17.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  29%|██▉       | 863/3000 [00:51<02:15, 15.74it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  29%|██▉       | 865/3000 [00:51<02:18, 15.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  29%|██▉       | 870/3000 [00:52<02:12, 16.04it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  29%|██▉       | 872/3000 [00:52<02:22, 14.96it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  29%|██▉       | 876/3000 [00:52<02:29, 14.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  29%|██▉       | 883/3000 [00:52<01:49, 19.42it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  30%|██▉       | 886/3000 [00:53<01:43, 20.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  30%|██▉       | 891/3000 [00:53<02:30, 14.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  30%|██▉       | 893/3000 [00:53<02:29, 14.09it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  30%|██▉       | 896/3000 [00:53<02:41, 13.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  30%|███       | 900/3000 [00:54<02:42, 12.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  30%|███       | 907/3000 [00:54<02:16, 15.37it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  30%|███       | 913/3000 [00:55<02:17, 15.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  31%|███       | 918/3000 [00:55<02:05, 16.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  31%|███       | 921/3000 [00:55<01:59, 17.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  31%|███       | 926/3000 [00:55<02:03, 16.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  31%|███       | 933/3000 [00:56<01:38, 20.97it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  31%|███       | 936/3000 [00:56<01:44, 19.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  31%|███▏      | 941/3000 [00:56<01:53, 18.14it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  32%|███▏      | 946/3000 [00:56<02:16, 15.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  32%|███▏      | 951/3000 [00:57<01:55, 17.66it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  32%|███▏      | 957/3000 [00:57<01:43, 19.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  32%|███▏      | 959/3000 [00:57<01:59, 17.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  32%|███▏      | 961/3000 [00:57<02:43, 12.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  32%|███▏      | 964/3000 [00:58<02:31, 13.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  32%|███▏      | 968/3000 [00:58<02:36, 12.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  33%|███▎      | 976/3000 [00:58<01:38, 20.55it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  33%|███▎      | 979/3000 [00:58<02:10, 15.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  33%|███▎      | 982/3000 [00:59<02:06, 16.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  33%|███▎      | 986/3000 [00:59<02:11, 15.35it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  33%|███▎      | 990/3000 [00:59<02:08, 15.60it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  33%|███▎      | 994/3000 [01:00<02:31, 13.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  33%|███▎      | 1000/3000 [01:00<02:15, 14.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  34%|███▎      | 1007/3000 [01:00<01:45, 18.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  34%|███▎      | 1012/3000 [01:01<02:11, 15.11it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  34%|███▍      | 1014/3000 [01:01<02:11, 15.16it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  34%|███▍      | 1019/3000 [01:01<02:03, 16.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  34%|███▍      | 1025/3000 [01:01<02:14, 14.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  34%|███▍      | 1029/3000 [01:02<02:04, 15.82it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  34%|███▍      | 1033/3000 [01:02<01:57, 16.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  35%|███▍      | 1039/3000 [01:02<01:46, 18.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  35%|███▍      | 1044/3000 [01:03<01:48, 18.02it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  35%|███▍      | 1048/3000 [01:03<01:28, 21.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  35%|███▌      | 1051/3000 [01:03<02:00, 16.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  35%|███▌      | 1053/3000 [01:03<02:16, 14.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  35%|███▌      | 1059/3000 [01:03<01:46, 18.29it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  35%|███▌      | 1062/3000 [01:04<01:40, 19.28it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  36%|███▌      | 1065/3000 [01:04<01:47, 18.05it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  36%|███▌      | 1072/3000 [01:04<01:35, 20.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  36%|███▌      | 1075/3000 [01:04<01:45, 18.17it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  36%|███▌      | 1080/3000 [01:05<02:16, 14.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  36%|███▌      | 1084/3000 [01:05<01:42, 18.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  36%|███▋      | 1089/3000 [01:05<01:56, 16.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  36%|███▋      | 1091/3000 [01:05<02:02, 15.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  37%|███▋      | 1098/3000 [01:06<01:51, 17.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  37%|███▋      | 1101/3000 [01:06<01:43, 18.37it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  37%|███▋      | 1106/3000 [01:06<01:57, 16.11it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  37%|███▋      | 1110/3000 [01:07<02:07, 14.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  37%|███▋      | 1120/3000 [01:07<01:43, 18.09it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  37%|███▋      | 1123/3000 [01:07<01:45, 17.80it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  38%|███▊      | 1129/3000 [01:08<01:39, 18.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  38%|███▊      | 1132/3000 [01:08<01:51, 16.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  38%|███▊      | 1138/3000 [01:08<01:55, 16.14it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  38%|███▊      | 1141/3000 [01:08<01:48, 17.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  38%|███▊      | 1147/3000 [01:09<01:38, 18.72it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  38%|███▊      | 1150/3000 [01:09<01:29, 20.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  39%|███▊      | 1158/3000 [01:09<01:25, 21.60it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  39%|███▉      | 1163/3000 [01:10<01:56, 15.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  39%|███▉      | 1168/3000 [01:10<01:43, 17.67it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  39%|███▉      | 1171/3000 [01:10<01:31, 20.04it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  39%|███▉      | 1174/3000 [01:10<01:26, 21.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  39%|███▉      | 1177/3000 [01:10<01:47, 17.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  39%|███▉      | 1181/3000 [01:11<02:03, 14.72it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  39%|███▉      | 1184/3000 [01:11<01:42, 17.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  40%|███▉      | 1191/3000 [01:11<01:29, 20.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  40%|███▉      | 1194/3000 [01:11<01:55, 15.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  40%|████      | 1200/3000 [01:12<01:46, 16.87it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  40%|████      | 1205/3000 [01:12<01:43, 17.38it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  40%|████      | 1212/3000 [01:12<01:28, 20.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  40%|████      | 1215/3000 [01:12<01:31, 19.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  41%|████      | 1221/3000 [01:13<01:44, 16.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  41%|████      | 1225/3000 [01:13<01:29, 19.80it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  41%|████      | 1230/3000 [01:13<01:38, 18.02it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  41%|████      | 1234/3000 [01:14<01:29, 19.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  41%|████      | 1237/3000 [01:14<01:45, 16.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  41%|████▏     | 1241/3000 [01:14<01:27, 20.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  41%|████▏     | 1244/3000 [01:14<01:51, 15.80it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  42%|████▏     | 1249/3000 [01:14<01:21, 21.38it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  42%|████▏     | 1252/3000 [01:15<01:41, 17.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  42%|████▏     | 1256/3000 [01:15<01:29, 19.52it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  42%|████▏     | 1259/3000 [01:15<01:38, 17.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  42%|████▏     | 1262/3000 [01:15<01:42, 16.88it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  42%|████▏     | 1269/3000 [01:16<01:36, 17.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  42%|████▏     | 1272/3000 [01:16<01:34, 18.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  43%|████▎     | 1276/3000 [01:16<01:49, 15.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  43%|████▎     | 1279/3000 [01:16<01:59, 14.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  43%|████▎     | 1285/3000 [01:17<01:58, 14.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  43%|████▎     | 1290/3000 [01:17<01:43, 16.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  43%|████▎     | 1294/3000 [01:17<01:30, 18.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  43%|████▎     | 1303/3000 [01:17<01:16, 22.28it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  44%|████▎     | 1306/3000 [01:18<01:31, 18.48it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  44%|████▎     | 1309/3000 [01:18<01:31, 18.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  44%|████▎     | 1311/3000 [01:18<01:51, 15.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  44%|████▍     | 1316/3000 [01:18<01:19, 21.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  44%|████▍     | 1322/3000 [01:19<01:26, 19.49it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  44%|████▍     | 1326/3000 [01:19<01:20, 20.74it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  44%|████▍     | 1334/3000 [01:19<01:28, 18.80it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  45%|████▍     | 1341/3000 [01:19<01:15, 22.07it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  45%|████▍     | 1344/3000 [01:20<01:33, 17.72it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  45%|████▌     | 1350/3000 [01:20<01:25, 19.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  45%|████▌     | 1357/3000 [01:20<01:11, 22.86it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  45%|████▌     | 1363/3000 [01:21<01:26, 18.84it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  46%|████▌     | 1370/3000 [01:21<01:09, 23.38it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  46%|████▌     | 1373/3000 [01:21<01:17, 20.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  46%|████▌     | 1379/3000 [01:21<01:24, 19.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  46%|████▌     | 1382/3000 [01:22<01:17, 21.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  46%|████▋     | 1388/3000 [01:22<01:15, 21.46it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  46%|████▋     | 1391/3000 [01:22<01:18, 20.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  47%|████▋     | 1397/3000 [01:22<01:17, 20.66it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  47%|████▋     | 1400/3000 [01:22<01:18, 20.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  47%|████▋     | 1405/3000 [01:23<01:29, 17.88it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  47%|████▋     | 1411/3000 [01:23<01:23, 18.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  47%|████▋     | 1415/3000 [01:23<01:08, 23.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  47%|████▋     | 1421/3000 [01:23<01:05, 24.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  47%|████▋     | 1424/3000 [01:24<01:19, 19.78it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  48%|████▊     | 1429/3000 [01:24<01:35, 16.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  48%|████▊     | 1439/3000 [01:24<00:56, 27.83it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  48%|████▊     | 1443/3000 [01:25<01:16, 20.37it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  48%|████▊     | 1449/3000 [01:25<01:30, 17.17it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  48%|████▊     | 1452/3000 [01:25<01:25, 18.07it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  49%|████▊     | 1461/3000 [01:26<01:06, 22.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  49%|████▉     | 1468/3000 [01:26<01:25, 17.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  49%|████▉     | 1471/3000 [01:26<01:21, 18.80it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  49%|████▉     | 1479/3000 [01:26<01:03, 23.88it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  49%|████▉     | 1482/3000 [01:27<01:20, 18.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  50%|████▉     | 1485/3000 [01:27<01:23, 18.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  50%|████▉     | 1492/3000 [01:27<01:20, 18.77it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  50%|████▉     | 1495/3000 [01:27<01:20, 18.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  50%|█████     | 1501/3000 [01:28<01:14, 20.09it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  50%|█████     | 1505/3000 [01:28<01:11, 20.83it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  50%|█████     | 1508/3000 [01:28<01:33, 15.96it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  50%|█████     | 1514/3000 [01:28<01:24, 17.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  51%|█████     | 1520/3000 [01:29<00:58, 25.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  51%|█████     | 1524/3000 [01:29<01:04, 23.05it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  51%|█████     | 1527/3000 [01:29<01:37, 15.11it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  51%|█████     | 1533/3000 [01:29<01:17, 18.97it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  51%|█████▏    | 1540/3000 [01:30<01:11, 20.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  52%|█████▏    | 1546/3000 [01:30<01:06, 21.82it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  52%|█████▏    | 1549/3000 [01:30<01:21, 17.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  52%|█████▏    | 1555/3000 [01:31<01:16, 18.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  52%|█████▏    | 1560/3000 [01:31<00:58, 24.60it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  52%|█████▏    | 1563/3000 [01:31<01:10, 20.29it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  52%|█████▏    | 1566/3000 [01:31<01:20, 17.72it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  52%|█████▏    | 1574/3000 [01:32<01:10, 20.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  53%|█████▎    | 1577/3000 [01:32<01:20, 17.57it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  53%|█████▎    | 1585/3000 [01:32<01:10, 20.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  53%|█████▎    | 1588/3000 [01:32<01:06, 21.29it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  53%|█████▎    | 1594/3000 [01:33<01:14, 18.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  53%|█████▎    | 1597/3000 [01:33<01:12, 19.40it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  54%|█████▎    | 1606/3000 [01:33<01:12, 19.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  54%|█████▎    | 1612/3000 [01:34<01:10, 19.77it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  54%|█████▍    | 1618/3000 [01:34<01:09, 20.02it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  54%|█████▍    | 1625/3000 [01:34<01:05, 20.87it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  54%|█████▍    | 1628/3000 [01:34<01:17, 17.71it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  54%|█████▍    | 1634/3000 [01:35<01:10, 19.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  55%|█████▍    | 1640/3000 [01:35<00:58, 23.09it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  55%|█████▍    | 1645/3000 [01:35<01:34, 14.32it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  55%|█████▌    | 1654/3000 [01:36<01:06, 20.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  55%|█████▌    | 1657/3000 [01:36<01:02, 21.61it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  55%|█████▌    | 1660/3000 [01:36<01:24, 15.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  56%|█████▌    | 1665/3000 [01:36<01:18, 17.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  56%|█████▌    | 1670/3000 [01:37<01:10, 18.78it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  56%|█████▌    | 1678/3000 [01:37<00:56, 23.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  56%|█████▌    | 1681/3000 [01:37<01:18, 16.87it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  56%|█████▋    | 1689/3000 [01:38<01:10, 18.57it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  56%|█████▋    | 1695/3000 [01:38<01:09, 18.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  57%|█████▋    | 1698/3000 [01:38<01:16, 16.92it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  57%|█████▋    | 1704/3000 [01:39<01:14, 17.48it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  57%|█████▋    | 1710/3000 [01:39<01:03, 20.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  57%|█████▋    | 1713/3000 [01:39<01:15, 17.09it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  57%|█████▋    | 1716/3000 [01:39<01:06, 19.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  57%|█████▋    | 1719/3000 [01:40<01:20, 15.86it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  58%|█████▊    | 1727/3000 [01:40<01:00, 20.92it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  58%|█████▊    | 1730/3000 [01:40<01:07, 18.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  58%|█████▊    | 1735/3000 [01:40<01:14, 16.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  58%|█████▊    | 1739/3000 [01:41<01:26, 14.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  58%|█████▊    | 1742/3000 [01:41<01:20, 15.66it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  58%|█████▊    | 1748/3000 [01:41<01:00, 20.72it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  58%|█████▊    | 1754/3000 [01:41<01:01, 20.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  59%|█████▊    | 1757/3000 [01:42<01:13, 16.96it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  59%|█████▉    | 1763/3000 [01:42<01:01, 20.16it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  59%|█████▉    | 1768/3000 [01:42<01:19, 15.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  59%|█████▉    | 1774/3000 [01:43<01:04, 19.14it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  59%|█████▉    | 1781/3000 [01:43<00:54, 22.48it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  59%|█████▉    | 1784/3000 [01:43<01:03, 19.11it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  60%|█████▉    | 1789/3000 [01:44<01:28, 13.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  60%|█████▉    | 1792/3000 [01:44<01:16, 15.84it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  60%|█████▉    | 1799/3000 [01:44<01:00, 19.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  60%|██████    | 1802/3000 [01:44<01:15, 15.92it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  60%|██████    | 1806/3000 [01:44<01:12, 16.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  60%|██████    | 1812/3000 [01:45<00:55, 21.46it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  60%|██████    | 1815/3000 [01:45<01:12, 16.40it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  61%|██████    | 1817/3000 [01:45<01:14, 15.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  61%|██████    | 1821/3000 [01:45<00:59, 19.67it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  61%|██████    | 1826/3000 [01:46<01:15, 15.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  61%|██████    | 1830/3000 [01:46<01:05, 17.93it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  61%|██████    | 1835/3000 [01:46<01:01, 18.96it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  61%|██████▏   | 1839/3000 [01:46<00:59, 19.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  61%|██████▏   | 1841/3000 [01:47<02:46,  6.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  62%|██████▏   | 1848/3000 [01:48<02:53,  6.66it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  62%|██████▏   | 1850/3000 [01:49<02:30,  7.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  62%|██████▏   | 1862/3000 [01:49<01:15, 15.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  62%|██████▏   | 1870/3000 [01:50<01:14, 15.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  62%|██████▏   | 1873/3000 [01:50<01:18, 14.32it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  63%|██████▎   | 1879/3000 [01:50<01:04, 17.27it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  63%|██████▎   | 1882/3000 [01:50<01:17, 14.49it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  63%|██████▎   | 1886/3000 [01:51<01:12, 15.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  63%|██████▎   | 1888/3000 [01:51<01:10, 15.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  63%|██████▎   | 1893/3000 [01:51<01:01, 18.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  63%|██████▎   | 1899/3000 [01:51<01:04, 17.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  63%|██████▎   | 1903/3000 [01:52<01:00, 18.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  64%|██████▎   | 1907/3000 [01:52<00:53, 20.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  64%|██████▎   | 1910/3000 [01:52<00:54, 20.06it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  64%|██████▍   | 1913/3000 [01:52<01:01, 17.71it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  64%|██████▍   | 1920/3000 [01:52<00:51, 20.92it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  64%|██████▍   | 1923/3000 [01:53<01:04, 16.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  64%|██████▍   | 1928/3000 [01:53<01:00, 17.71it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  64%|██████▍   | 1931/3000 [01:53<01:10, 15.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  64%|██████▍   | 1934/3000 [01:53<01:05, 16.17it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  65%|██████▍   | 1941/3000 [01:54<00:50, 21.11it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  65%|██████▍   | 1944/3000 [01:54<01:12, 14.52it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  65%|██████▍   | 1947/3000 [01:54<01:15, 13.97it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  65%|██████▌   | 1953/3000 [01:54<00:57, 18.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  65%|██████▌   | 1956/3000 [01:55<00:52, 19.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  65%|██████▌   | 1962/3000 [01:55<01:04, 16.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  66%|██████▌   | 1966/3000 [01:55<01:13, 14.04it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  66%|██████▌   | 1972/3000 [01:56<00:55, 18.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  66%|██████▌   | 1975/3000 [01:56<00:50, 20.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  66%|██████▌   | 1978/3000 [01:56<00:59, 17.27it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  66%|██████▌   | 1984/3000 [01:56<00:57, 17.67it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  66%|██████▌   | 1986/3000 [01:57<01:14, 13.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  66%|██████▋   | 1994/3000 [01:57<00:50, 19.80it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  67%|██████▋   | 1997/3000 [01:57<00:57, 17.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  67%|██████▋   | 2001/3000 [01:57<01:03, 15.66it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  67%|██████▋   | 2004/3000 [01:58<01:05, 15.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  67%|██████▋   | 2010/3000 [01:58<01:00, 16.48it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  67%|██████▋   | 2016/3000 [01:58<00:48, 20.09it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  67%|██████▋   | 2021/3000 [01:59<01:09, 14.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  68%|██████▊   | 2026/3000 [01:59<01:00, 16.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  68%|██████▊   | 2030/3000 [01:59<01:02, 15.55it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  68%|██████▊   | 2035/3000 [02:00<00:58, 16.49it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  68%|██████▊   | 2040/3000 [02:00<00:54, 17.58it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  68%|██████▊   | 2046/3000 [02:00<00:50, 18.87it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  68%|██████▊   | 2048/3000 [02:00<00:51, 18.40it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  68%|██████▊   | 2054/3000 [02:01<01:02, 15.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  69%|██████▊   | 2057/3000 [02:01<00:54, 17.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  69%|██████▊   | 2062/3000 [02:01<01:00, 15.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  69%|██████▉   | 2064/3000 [02:02<01:10, 13.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  69%|██████▉   | 2068/3000 [02:02<01:14, 12.47it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  69%|██████▉   | 2074/3000 [02:02<00:50, 18.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  69%|██████▉   | 2076/3000 [02:02<01:13, 12.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  69%|██████▉   | 2078/3000 [02:03<01:09, 13.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  69%|██████▉   | 2082/3000 [02:03<01:15, 12.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  70%|██████▉   | 2086/3000 [02:03<00:56, 16.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  70%|██████▉   | 2090/3000 [02:03<00:45, 19.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  70%|██████▉   | 2095/3000 [02:04<01:03, 14.29it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  70%|██████▉   | 2097/3000 [02:04<00:59, 15.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  70%|███████   | 2105/3000 [02:04<00:48, 18.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metricsmetrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      " {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  70%|███████   | 2108/3000 [02:04<00:55, 16.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  70%|███████   | 2114/3000 [02:05<00:54, 16.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  71%|███████   | 2118/3000 [02:05<00:55, 15.84it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  71%|███████   | 2120/3000 [02:05<00:53, 16.46it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  71%|███████   | 2125/3000 [02:06<00:55, 15.83it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  71%|███████   | 2127/3000 [02:06<00:57, 15.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  71%|███████   | 2132/3000 [02:06<00:58, 14.80it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  71%|███████   | 2135/3000 [02:06<00:48, 17.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  71%|███████▏  | 2140/3000 [02:07<00:54, 15.72it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  71%|███████▏  | 2144/3000 [02:07<00:55, 15.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  72%|███████▏  | 2151/3000 [02:07<00:46, 18.07it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  72%|███████▏  | 2155/3000 [02:07<00:52, 16.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  72%|███████▏  | 2160/3000 [02:08<00:58, 14.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  72%|███████▏  | 2165/3000 [02:08<00:54, 15.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  72%|███████▏  | 2170/3000 [02:08<00:52, 15.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  72%|███████▏  | 2174/3000 [02:09<00:58, 14.05it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  73%|███████▎  | 2176/3000 [02:09<01:03, 13.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  73%|███████▎  | 2183/3000 [02:09<00:40, 19.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  73%|███████▎  | 2186/3000 [02:09<00:46, 17.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  73%|███████▎  | 2188/3000 [02:10<00:50, 16.11it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  73%|███████▎  | 2193/3000 [02:10<00:56, 14.37it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  73%|███████▎  | 2197/3000 [02:10<00:41, 19.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  73%|███████▎  | 2202/3000 [02:11<00:52, 15.27it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  74%|███████▎  | 2211/3000 [02:11<00:44, 17.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  74%|███████▍  | 2214/3000 [02:11<00:46, 17.06it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  74%|███████▍  | 2218/3000 [02:11<00:49, 15.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  74%|███████▍  | 2225/3000 [02:12<00:47, 16.16it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  74%|███████▍  | 2230/3000 [02:12<00:36, 21.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  74%|███████▍  | 2233/3000 [02:12<00:36, 21.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  75%|███████▍  | 2236/3000 [02:13<00:47, 16.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  75%|███████▍  | 2241/3000 [02:13<00:56, 13.48it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  75%|███████▍  | 2244/3000 [02:13<00:54, 13.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  75%|███████▍  | 2247/3000 [02:13<00:46, 16.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  75%|███████▌  | 2250/3000 [02:13<00:43, 17.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  75%|███████▌  | 2255/3000 [02:14<00:45, 16.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  75%|███████▌  | 2259/3000 [02:14<00:57, 12.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  75%|███████▌  | 2262/3000 [02:14<00:49, 14.84it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  76%|███████▌  | 2266/3000 [02:14<00:40, 17.96it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  76%|███████▌  | 2271/3000 [02:15<00:37, 19.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  76%|███████▌  | 2276/3000 [02:15<00:48, 15.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  76%|███████▌  | 2278/3000 [02:15<00:46, 15.60it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  76%|███████▌  | 2284/3000 [02:16<00:45, 15.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  76%|███████▋  | 2289/3000 [02:16<00:33, 21.32it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  76%|███████▋  | 2292/3000 [02:16<00:42, 16.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  76%|███████▋  | 2294/3000 [02:16<00:42, 16.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  77%|███████▋  | 2301/3000 [02:17<00:37, 18.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  77%|███████▋  | 2307/3000 [02:17<00:31, 22.02it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  77%|███████▋  | 2310/3000 [02:17<00:37, 18.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  77%|███████▋  | 2315/3000 [02:18<00:55, 12.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  77%|███████▋  | 2321/3000 [02:18<00:37, 18.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  77%|███████▋  | 2324/3000 [02:18<00:38, 17.40it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  78%|███████▊  | 2327/3000 [02:18<00:42, 15.88it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  78%|███████▊  | 2329/3000 [02:19<00:51, 13.02it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  78%|███████▊  | 2337/3000 [02:19<00:34, 18.96it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  78%|███████▊  | 2341/3000 [02:19<00:29, 22.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  78%|███████▊  | 2344/3000 [02:19<00:35, 18.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  78%|███████▊  | 2349/3000 [02:20<00:45, 14.32it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  79%|███████▊  | 2357/3000 [02:20<00:30, 20.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  79%|███████▊  | 2360/3000 [02:20<00:34, 18.60it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  79%|███████▉  | 2367/3000 [02:21<00:42, 14.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  79%|███████▉  | 2376/3000 [02:21<00:31, 19.58it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  79%|███████▉  | 2379/3000 [02:22<00:44, 14.09it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  80%|███████▉  | 2385/3000 [02:22<00:36, 16.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  80%|███████▉  | 2391/3000 [02:22<00:36, 16.82it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  80%|███████▉  | 2394/3000 [02:22<00:38, 15.74it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  80%|███████▉  | 2399/3000 [02:23<00:42, 14.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  80%|████████  | 2403/3000 [02:23<00:33, 17.89it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  80%|████████  | 2407/3000 [02:23<00:27, 21.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  80%|████████  | 2410/3000 [02:23<00:32, 18.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  80%|████████  | 2415/3000 [02:24<00:34, 16.78it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  81%|████████  | 2420/3000 [02:24<00:30, 19.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  81%|████████  | 2424/3000 [02:24<00:27, 21.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  81%|████████  | 2429/3000 [02:24<00:36, 15.47it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  81%|████████  | 2435/3000 [02:25<00:26, 21.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  81%|████████▏ | 2441/3000 [02:25<00:19, 28.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  82%|████████▏ | 2445/3000 [02:25<00:23, 23.48it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  82%|████████▏ | 2452/3000 [02:26<00:29, 18.36it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  82%|████████▏ | 2459/3000 [02:26<00:24, 21.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  82%|████████▏ | 2462/3000 [02:26<00:24, 22.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  82%|████████▏ | 2468/3000 [02:26<00:33, 16.11it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  82%|████████▎ | 2475/3000 [02:27<00:26, 19.58it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  83%|████████▎ | 2479/3000 [02:27<00:23, 22.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  83%|████████▎ | 2482/3000 [02:27<00:27, 18.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  83%|████████▎ | 2485/3000 [02:27<00:26, 19.17it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metricsmetrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      " {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  83%|████████▎ | 2491/3000 [02:28<00:28, 17.57it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  83%|████████▎ | 2495/3000 [02:28<00:29, 17.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  83%|████████▎ | 2500/3000 [02:28<00:26, 18.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  83%|████████▎ | 2504/3000 [02:28<00:28, 17.46it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  84%|████████▎ | 2508/3000 [02:28<00:22, 21.93it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  84%|████████▍ | 2515/3000 [02:29<00:24, 19.58it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  84%|████████▍ | 2520/3000 [02:29<00:19, 24.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  84%|████████▍ | 2523/3000 [02:29<00:23, 20.04it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  84%|████████▍ | 2529/3000 [02:30<00:26, 17.47it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  85%|████████▍ | 2536/3000 [02:30<00:20, 22.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  85%|████████▍ | 2539/3000 [02:30<00:19, 23.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  85%|████████▍ | 2542/3000 [02:30<00:25, 17.74it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  85%|████████▍ | 2545/3000 [02:30<00:27, 16.83it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  85%|████████▌ | 2551/3000 [02:31<00:26, 17.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  85%|████████▌ | 2556/3000 [02:31<00:19, 22.63it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  86%|████████▌ | 2565/3000 [02:31<00:19, 21.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  86%|████████▌ | 2568/3000 [02:32<00:25, 17.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  86%|████████▌ | 2571/3000 [02:32<00:22, 19.27it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  86%|████████▌ | 2579/3000 [02:32<00:19, 22.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  86%|████████▌ | 2582/3000 [02:32<00:19, 21.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  86%|████████▌ | 2585/3000 [02:33<00:25, 16.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  86%|████████▋ | 2590/3000 [02:33<00:30, 13.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  87%|████████▋ | 2598/3000 [02:33<00:18, 21.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  87%|████████▋ | 2601/3000 [02:33<00:19, 20.67it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  87%|████████▋ | 2604/3000 [02:34<00:19, 20.29it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  87%|████████▋ | 2610/3000 [02:34<00:22, 17.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  87%|████████▋ | 2617/3000 [02:34<00:18, 21.11it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  87%|████████▋ | 2623/3000 [02:35<00:19, 18.97it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  88%|████████▊ | 2626/3000 [02:35<00:24, 15.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  88%|████████▊ | 2632/3000 [02:35<00:16, 22.37it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  88%|████████▊ | 2638/3000 [02:35<00:17, 21.17it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  88%|████████▊ | 2641/3000 [02:36<00:20, 17.36it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  88%|████████▊ | 2646/3000 [02:36<00:21, 16.83it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  88%|████████▊ | 2652/3000 [02:36<00:18, 18.37it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  88%|████████▊ | 2654/3000 [02:36<00:22, 15.57it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  89%|████████▊ | 2658/3000 [02:37<00:23, 14.28it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  89%|████████▊ | 2661/3000 [02:37<00:20, 16.78it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  89%|████████▉ | 2664/3000 [02:37<00:19, 17.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  89%|████████▉ | 2668/3000 [02:37<00:18, 17.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  89%|████████▉ | 2670/3000 [02:38<00:23, 14.02it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  89%|████████▉ | 2678/3000 [02:38<00:16, 19.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  89%|████████▉ | 2681/3000 [02:38<00:16, 19.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  89%|████████▉ | 2684/3000 [02:38<00:20, 15.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  90%|████████▉ | 2689/3000 [02:39<00:21, 14.66it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  90%|████████▉ | 2693/3000 [02:39<00:16, 18.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  90%|████████▉ | 2699/3000 [02:39<00:17, 17.52it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  90%|█████████ | 2701/3000 [02:40<00:22, 13.09it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  90%|█████████ | 2705/3000 [02:40<00:22, 13.38it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  90%|█████████ | 2710/3000 [02:40<00:16, 17.07it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  90%|█████████ | 2715/3000 [02:40<00:17, 16.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  91%|█████████ | 2718/3000 [02:40<00:15, 17.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  91%|█████████ | 2723/3000 [02:41<00:18, 14.92it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  91%|█████████ | 2727/3000 [02:41<00:13, 19.83it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  91%|█████████ | 2730/3000 [02:41<00:15, 17.63it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  91%|█████████ | 2733/3000 [02:41<00:17, 15.36it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  91%|█████████▏| 2739/3000 [02:42<00:16, 15.36it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  91%|█████████▏| 2741/3000 [02:42<00:19, 13.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  92%|█████████▏| 2746/3000 [02:42<00:17, 14.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  92%|█████████▏| 2754/3000 [02:43<00:13, 18.05it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  92%|█████████▏| 2757/3000 [02:43<00:13, 18.05it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  92%|█████████▏| 2761/3000 [02:43<00:16, 14.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  92%|█████████▏| 2764/3000 [02:43<00:15, 15.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  92%|█████████▏| 2768/3000 [02:44<00:14, 15.86it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  92%|█████████▏| 2774/3000 [02:44<00:10, 20.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  93%|█████████▎| 2777/3000 [02:44<00:12, 17.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  93%|█████████▎| 2782/3000 [02:44<00:14, 15.42it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  93%|█████████▎| 2785/3000 [02:45<00:13, 15.71it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  93%|█████████▎| 2787/3000 [02:45<00:13, 16.38it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  93%|█████████▎| 2792/3000 [02:45<00:13, 15.77it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  93%|█████████▎| 2794/3000 [02:45<00:16, 12.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  93%|█████████▎| 2798/3000 [02:46<00:13, 14.77it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  93%|█████████▎| 2803/3000 [02:46<00:12, 16.32it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  94%|█████████▎| 2806/3000 [02:46<00:11, 17.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  94%|█████████▍| 2813/3000 [02:46<00:10, 17.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  94%|█████████▍| 2816/3000 [02:47<00:10, 16.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  94%|█████████▍| 2821/3000 [02:47<00:10, 16.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  94%|█████████▍| 2824/3000 [02:47<00:10, 16.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  94%|█████████▍| 2828/3000 [02:47<00:10, 15.78it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  94%|█████████▍| 2831/3000 [02:48<00:09, 17.78it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  95%|█████████▍| 2837/3000 [02:48<00:08, 19.46it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  95%|█████████▍| 2841/3000 [02:48<00:09, 17.57it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  95%|█████████▍| 2848/3000 [02:49<00:09, 16.28it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  95%|█████████▌| 2854/3000 [02:49<00:07, 19.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  95%|█████████▌| 2860/3000 [02:49<00:07, 19.11it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  95%|█████████▌| 2863/3000 [02:50<00:10, 13.67it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  96%|█████████▌| 2869/3000 [02:50<00:08, 15.36it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  96%|█████████▌| 2873/3000 [02:50<00:08, 15.63it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  96%|█████████▌| 2878/3000 [02:50<00:05, 20.92it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  96%|█████████▌| 2884/3000 [02:51<00:05, 21.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  96%|█████████▋| 2890/3000 [02:51<00:07, 15.28it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  96%|█████████▋| 2894/3000 [02:51<00:07, 13.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  97%|█████████▋| 2897/3000 [02:52<00:06, 15.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  97%|█████████▋| 2901/3000 [02:52<00:07, 13.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  97%|█████████▋| 2905/3000 [02:52<00:06, 14.82it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  97%|█████████▋| 2910/3000 [02:53<00:06, 14.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  97%|█████████▋| 2915/3000 [02:53<00:05, 16.86it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  97%|█████████▋| 2920/3000 [02:53<00:04, 19.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  97%|█████████▋| 2923/3000 [02:53<00:04, 16.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  98%|█████████▊| 2928/3000 [02:54<00:04, 15.61it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  98%|█████████▊| 2930/3000 [02:54<00:04, 15.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  98%|█████████▊| 2935/3000 [02:54<00:04, 15.72it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  98%|█████████▊| 2940/3000 [02:54<00:03, 19.87it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  98%|█████████▊| 2943/3000 [02:55<00:04, 14.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  98%|█████████▊| 2948/3000 [02:55<00:03, 15.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  98%|█████████▊| 2952/3000 [02:55<00:02, 18.71it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  99%|█████████▊| 2959/3000 [02:55<00:01, 21.52it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  99%|█████████▊| 2962/3000 [02:56<00:02, 14.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  99%|█████████▉| 2964/3000 [02:56<00:02, 14.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  99%|█████████▉| 2969/3000 [02:56<00:01, 16.36it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow:  99%|█████████▉| 2974/3000 [02:56<00:01, 22.89it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow:  99%|█████████▉| 2980/3000 [02:57<00:01, 16.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow: 100%|█████████▉| 2986/3000 [02:57<00:00, 15.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow: 100%|█████████▉| 2993/3000 [02:57<00:00, 20.02it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow: 100%|█████████▉| 2996/3000 [02:58<00:00, 17.09it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluating workflow: 100%|█████████▉| 2998/3000 [02:58<00:00, 13.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow: 100%|█████████▉| 2999/3000 [03:13<00:02,  2.03s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluating workflow: 100%|██████████| 3000/3000 [03:13<00:00, 15.48it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n",
      "\u001b[32m2026-01-04 11:00:08.299\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36m__main__\u001b[0m:\u001b[36m<module>\u001b[0m:\u001b[36m4\u001b[0m - \u001b[1mEvaluation metrics (before optimization): {'f1': 0.0, 'em': 0.0, 'acc': 0.7566666666666667}\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "logger.info(\"Evaluating workflow on test set...\")\n",
    "with suppress_logger_info():\n",
    "    results = textgrad_optimizer.evaluate(dataset=benchmark, eval_mode=\"test\")\n",
    "logger.info(f\"Evaluation metrics (before optimization): {results}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "f3cdb546",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'f1': 0.0, 'em': 0.0, 'acc': 0.7566666666666667}"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3959377a",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7132c3af",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "c1bc0e23",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "df = pd.read_csv(\"../../aflow_save.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "d81e6d2e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Unnamed: 0    247.500000\n",
       "f1              0.683997\n",
       "em              0.532258\n",
       "acc             0.627016\n",
       "dtype: float64"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.iloc[0:496].mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "6d925c5b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "496"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "1489//3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a36220ca",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}